Retain processing evidence only for selected outputs

This commit is contained in:
2026-11-16 15:08:02 +00:00
parent 5167154a1c
commit 891d4aac2c
21 changed files with 494 additions and 229 deletions

View File

@@ -207,7 +207,7 @@ func TestRunnerRetainsLaterProposalEvidenceWhenEarlierSectionFails(t *testing.T)
cfg.Chunking.MaxSectionTokens = 3
cfg.Chunking.MinSectionTokens = 0
out, err := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}}).Run(ctx, RunInput{
Config: &cfg, CaptureCandidates: true,
Config: &cfg, Evidence: &EvidenceSelection{Reports: true, Debug: true},
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "original text"}, {ID: 93, Text: "other text"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}},
})

View File

@@ -0,0 +1,94 @@
package runner
import (
"context"
"reflect"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
)
func evidenceFixture(size int) (*Runner, RunInput) {
cfg := runnerTestConfig()
cfg.Chunking.MaxSectionTokens = 1
cfg.Chunking.MinSectionTokens = 0
transcript := &schema.Transcript{Segments: make([]schema.Segment, size)}
for i := range transcript.Segments {
transcript.Segments[i] = schema.Segment{ID: i + 1, Text: "word sample"}
}
validator := contextValidator{validate: func(_ context.Context, req validators.Request) (validators.Result, error) {
var decisions []validators.Decision
for _, candidate := range req.CandidateProposal {
decisions = append(decisions, validators.Decision{ProposalIndex: candidate.ProposalIndex, Approved: true, Disposition: validators.DispositionApproved})
}
return validators.Result{Decisions: decisions}, nil
}}
module := outcomeModule{
fakeModule: fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{validator}},
proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
operation := outcomes.Operation{Reference: outcomes.Reference{Kind: outcomes.OperationProposal}}
category := req.Section.Index % 4
if category == 0 {
operation.FailureClass, operation.FailureReason = outcomes.FailureRecoverable, outcomes.ReasonTransportFailed
return contracts.ProposalResult{Operations: []outcomes.Operation{operation}}, nil
}
operation.Succeeded = true
original := "word"
if category == 1 {
original = "absent"
}
if category == 2 {
operation.RecoveryReasons = []outcomes.RecoveryReason{outcomes.RecoveryServiceRecovered}
}
return contracts.ProposalResult{
Operations: []outcomes.Operation{operation},
Proposals: []proposals.CorrectionProposal{{TargetSegmentID: req.Section.StartSegmentID, OriginalText: original, CorrectedText: "Word"}},
}, nil
},
}
return New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}}), RunInput{
Config: &cfg, Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}},
}
}
func TestEvidenceSelectionKeepsTotalsAndRequestedRecords(t *testing.T) {
r, input := evidenceFixture(40)
want := outcomes.WorkTotals{Total: 40, Started: 40, Completed: 40, WithErrors: 10, Corrections: 20,
ModelOperations: 40, ProposalResponses: 30, Recoveries: 20, FilteringResponses: 10}
var baseline *schema.Transcript
for _, tc := range []struct {
name string
selection EvidenceSelection
units int
candidates int
}{
{name: "summary"},
{name: "reports", selection: EvidenceSelection{Reports: true}, units: 40},
{name: "debug", selection: EvidenceSelection{Debug: true}, candidates: 30},
{name: "reports and debug", selection: EvidenceSelection{Reports: true, Debug: true}, units: 40, candidates: 30},
} {
t.Run(tc.name, func(t *testing.T) {
input.Evidence = &tc.selection
out, err := r.Run(context.Background(), input)
if err != nil {
t.Fatal(err)
}
if out.Totals == nil || *out.Totals != want {
t.Fatalf("totals: got %+v, want %+v", out.Totals, want)
}
if len(out.WorkUnits) != tc.units || len(out.Candidates) != tc.candidates ||
len(out.ModuleResults[0].ValidatorDecisions) != tc.candidates {
t.Fatalf("retained units=%d candidates=%d decisions=%d", len(out.WorkUnits), len(out.Candidates), len(out.ModuleResults[0].ValidatorDecisions))
}
if baseline == nil {
baseline = out.FinalTranscript
} else if !reflect.DeepEqual(baseline, out.FinalTranscript) {
t.Fatal("output selection changed final corrections")
}
})
}
}

View File

@@ -161,7 +161,7 @@ func runRepairBranch(ctx context.Context, branchInput repairBranchInput) (repair
}
proposalIndex, attemptNumber := original.ProposalIndex, branch.AttemptCount+1
return &structuredoutput.Request{PromptID: promptID, Variables: map[string]string{"RepairRequestJSON": payload}, Schema: responseschema.CorrectionRepairKey, Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex, ProposalIndex: &proposalIndex, RepairAttempt: &attemptNumber}}, nil
return &structuredoutput.Request{PromptID: promptID, Variables: map[string]string{"RepairRequestJSON": payload}, Schema: responseschema.CorrectionRepairKey, CaptureDiagnostics: debug, Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex, ProposalIndex: &proposalIndex, RepairAttempt: &attemptNumber}}, nil
}
exhausted, coordinationErr := input.Spec.RepairCoordinator.Coordinate(ctx, input.Exchange, input.Spec.Settings.LLM.CorrectionRepairAttempts, func(ctx context.Context, turn structuredoutput.Turn) (*structuredoutput.Request, error) {
if turn.Response == nil {

View File

@@ -45,8 +45,6 @@ type ModuleResult struct {
ModuleInstance string `json:"module_instance"`
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
Status string `json:"status"`
ProposalCount int `json:"proposal_count"`
Operations []outcomes.Operation `json:"operations,omitempty"`
ValidatorDecisions []ValidatorDecisionRecord `json:"validator_decisions,omitempty"`
ValidatorRejected []ValidatorRejectedChange `json:"validator_rejected,omitempty"`
AppliedChanges []proposals.AppliedChange `json:"applied_changes,omitempty"`
@@ -95,13 +93,32 @@ type ValidatorRejectedChange struct {
Message string `json:"message"`
}
// EvidenceSelection selects the detailed views consumed by the invocation.
type EvidenceSelection struct {
Reports bool
Debug bool
}
type evidenceNeeds struct {
reports bool
decisions bool
candidates bool
}
func selectEvidence(selection *EvidenceSelection) evidenceNeeds {
if selection == nil {
return evidenceNeeds{reports: true, decisions: true}
}
return evidenceNeeds{reports: selection.Reports, decisions: selection.Debug, candidates: selection.Debug}
}
// RunInput is the deterministic runner input.
type RunInput struct {
Config *config.Config
Transcript *schema.Transcript
ModuleSpecs []contracts.ModuleRunSpec
Interactions structuredoutput.InteractionSink
CaptureCandidates bool
Config *config.Config
Transcript *schema.Transcript
ModuleSpecs []contracts.ModuleRunSpec
Interactions structuredoutput.InteractionSink
Evidence *EvidenceSelection
}
// RunOutput is the deterministic runner output.
@@ -109,6 +126,7 @@ type RunOutput struct {
FinalTranscript *schema.Transcript `json:"-"`
ModuleResults []ModuleResult `json:"module_results"`
Sections []contracts.SectionMetadata `json:"sections"`
Totals *outcomes.WorkTotals `json:"-"`
InventoryAvailable bool `json:"-"`
WorkUnits []WorkUnitOutcome `json:"work_units"`
Candidates []proposals.EnrichedCorrectionProposal `json:"-"`
@@ -132,54 +150,70 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
if err != nil {
return RunOutput{FinalTranscript: working}, &ExecutionFailure{Phase: "chunking", Cause: fmt.Errorf("chunk normalized transcript: %w", err)}
}
inventory := make([]contracts.SectionMetadata, len(sections))
workUnits := make([]WorkUnitOutcome, 0, len(input.ModuleSpecs)*len(sections))
evidence := selectEvidence(input.Evidence)
totals := outcomes.WorkTotals{Total: len(input.ModuleSpecs) * len(sections)}
var inventory []contracts.SectionMetadata
var workUnits []WorkUnitOutcome
var candidates []proposals.EnrichedCorrectionProposal
for i, section := range sections {
inventory[i] = contracts.SectionMetadataFromSection(section)
}
for moduleIndex, spec := range input.ModuleSpecs {
for _, section := range inventory {
workUnits = append(workUnits, WorkUnitOutcome{
ModuleIndex: moduleIndex, StageIndex: spec.Index, StageID: spec.ID, ModuleKey: spec.ModuleKey,
ModuleInstance: spec.InstanceName, Section: section,
State: outcomes.WorkNotStarted,
})
if evidence.reports {
inventory = make([]contracts.SectionMetadata, len(sections))
for i, section := range sections {
inventory[i] = contracts.SectionMetadataFromSection(section)
}
workUnits = make([]WorkUnitOutcome, 0, totals.Total)
for moduleIndex, spec := range input.ModuleSpecs {
for _, section := range inventory {
workUnits = append(workUnits, newWorkUnit(spec, section, moduleIndex))
}
}
}
snapshot := func() RunOutput {
return RunOutput{
FinalTranscript: transcriptFromSections(sections), ModuleResults: results,
Sections: inventory, InventoryAvailable: true, WorkUnits: workUnits, Candidates: candidates,
Sections: inventory, InventoryAvailable: true, Totals: &totals, WorkUnits: workUnits, Candidates: candidates,
}
}
for moduleIndex, spec := range input.ModuleSpecs {
execution, err := r.executeModule(ctx, spec, sections, input.Interactions)
units := workUnits[moduleIndex*len(sections) : (moduleIndex+1)*len(sections)]
copy(units, execution.pipeline.WorkUnits)
if err == nil && ctx.Err() != nil {
err = locateFailure(ctx.Err(), "application", failureReference(spec, nil, ""), false, nil)
var reportUnits []WorkUnitOutcome
if evidence.reports {
reportUnits = workUnits[moduleIndex*len(sections) : (moduleIndex+1)*len(sections)]
}
execution, err := r.executeModule(ctx, spec, sections, input.Interactions, evidence, reportUnits)
units := execution.pipeline.WorkUnits
if err == nil {
previous := sections
sections = execution.updated
attributeApplyFiltering(previous, execution.pipeline.Approved, execution.skipped, execution.policy, units)
filters := findApplyFiltering(sections, execution.pipeline.Approved, execution.skipped, execution.policy)
if canceled := ctx.Err(); canceled != nil {
err = locateFailure(canceled, "application", failureReference(spec, nil, ""), false, nil)
} else {
execution.pipeline.recordApplyFiltering(filters, evidence.reports)
sections = execution.updated
execution.pipeline.Totals.Started = len(sections)
execution.pipeline.Totals.Completed = len(sections)
execution.pipeline.Totals.Corrections = len(execution.applied)
}
}
var failure *ExecutionFailure
applicationInterrupted := errors.As(err, &failure) && failure.Phase == "application"
appliedIndices := make(map[int]struct{}, len(execution.applied))
for _, change := range execution.applied {
appliedIndices[change.ProposalIndex] = struct{}{}
if applicationInterrupted {
execution.pipeline.Totals.Completed = 0
}
totals.Add(execution.pipeline.Totals)
appliedIndices := make(map[int]struct{})
if evidence.reports || evidence.candidates {
for _, change := range execution.applied {
appliedIndices[change.ProposalIndex] = struct{}{}
}
}
var operations []outcomes.Operation
for i := range units {
unit := &units[i]
unit.ModuleIndex = moduleIndex
if err == nil {
unit.State = outcomes.WorkCompleted
unit.CorrectionsApplied = execution.appliedBySection[i]
if evidence.reports {
unit.CorrectionsApplied = execution.appliedBySection[unit.Section.Index]
}
} else if applicationInterrupted && unit.State != outcomes.WorkNotStarted {
unit.State = outcomes.WorkInterrupted
}
@@ -196,16 +230,14 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
branch.Outcome = "approved_skipped"
}
}
operations = append(operations, unit.Operations...)
}
if input.CaptureCandidates {
candidates = append(candidates, execution.pipeline.Candidates...)
if !evidence.reports && evidence.candidates {
workUnits = append(workUnits, units...)
}
candidates = append(candidates, execution.pipeline.Candidates...)
result := ModuleResult{
StageIndex: spec.Index, StageID: spec.ID, ModuleKey: spec.ModuleKey,
ModuleInstance: spec.InstanceName, ReplacementPolicy: execution.policy,
ProposalCount: execution.pipeline.ProposalCount, Operations: operations,
ValidatorDecisions: execution.pipeline.ValidatorDecisions, ValidatorRejected: execution.pipeline.ValidatorRejected,
}
if err != nil {
@@ -213,8 +245,10 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (RunOutput, error) {
result.ErrorMessage = err.Error()
} else {
result.Status = ModuleStatusSuccess
result.AppliedChanges = execution.applied
result.SkippedChanges = execution.skipped
if evidence.decisions {
result.AppliedChanges = execution.applied
result.SkippedChanges = execution.skipped
}
}
results = append(results, result)
if err != nil {
@@ -237,7 +271,7 @@ type moduleExecution struct {
appliedBySection []int
}
func (r *Runner) executeModule(ctx context.Context, spec contracts.ModuleRunSpec, sections []chunking.Section, interactions structuredoutput.InteractionSink) (moduleExecution, error) {
func (r *Runner) executeModule(ctx context.Context, spec contracts.ModuleRunSpec, sections []chunking.Section, interactions structuredoutput.InteractionSink, evidence evidenceNeeds, reportUnits []WorkUnitOutcome) (moduleExecution, error) {
var execution moduleExecution
origin := failureReference(spec, nil, "")
if err := ctx.Err(); err != nil {
@@ -253,6 +287,8 @@ func (r *Runner) executeModule(ctx context.Context, spec contracts.ModuleRunSpec
}
execution.pipeline, err = runModulePipeline(ctx, collectSectionProposalsInput{
Module: module,
Evidence: &evidence,
ReportUnits: reportUnits,
Spec: spec,
Policy: execution.policy,
Sections: sections,
@@ -323,6 +359,8 @@ func applyWithinSections(sections []chunking.Section, approved []proposals.Enric
}
type collectSectionProposalsInput struct {
Evidence *evidenceNeeds
ReportUnits []WorkUnitOutcome
Interactions structuredoutput.InteractionSink
Module contracts.TranscriptModule
Spec contracts.ModuleRunSpec
@@ -356,16 +394,18 @@ type sectionValidationResult struct {
}
type sectionWork struct {
unit WorkUnitOutcome
candidates []proposals.EnrichedCorrectionProposal
validation *sectionValidationResult
finished bool
}
type modulePipelineResult struct {
ProposalCount int
Totals outcomes.WorkTotals
proposalFacts map[int]proposalRecovery
revised map[int]struct{}
Candidates []proposals.EnrichedCorrectionProposal
Approved []proposals.EnrichedCorrectionProposal
Operations []outcomes.Operation
WorkUnits []WorkUnitOutcome
ValidatorDecisions []ValidatorDecisionRecord
ValidatorRejected []ValidatorRejectedChange
@@ -410,17 +450,16 @@ func proposeSection(ctx context.Context, input collectSectionProposalsInput, sec
}
func runModulePipeline(ctx context.Context, input collectSectionProposalsInput) (modulePipelineResult, error) {
out := modulePipelineResult{
Approved: make([]proposals.EnrichedCorrectionProposal, 0),
Operations: make([]outcomes.Operation, 0),
WorkUnits: make([]WorkUnitOutcome, len(input.Sections)),
ValidatorDecisions: make([]ValidatorDecisionRecord, 0),
ValidatorRejected: make([]ValidatorRejectedChange, 0),
var out modulePipelineResult
evidence := selectEvidence(nil)
if input.Evidence != nil {
evidence = *input.Evidence
}
for i, section := range input.Sections {
out.WorkUnits[i] = WorkUnitOutcome{
ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, StageIndex: input.Spec.Index, StageID: input.Spec.ID,
Section: contracts.SectionMetadataFromSection(section), State: outcomes.WorkNotStarted,
out.WorkUnits = input.ReportUnits
if evidence.reports && out.WorkUnits == nil {
out.WorkUnits = make([]WorkUnitOutcome, len(input.Sections))
for i, section := range input.Sections {
out.WorkUnits[i] = newWorkUnit(input.Spec, contracts.SectionMetadataFromSection(section), 0)
}
}
@@ -487,17 +526,17 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
return sectionEnriched
}
recordCandidates := func(sectionPos int, enriched []proposals.EnrichedCorrectionProposal) {
unit := &out.WorkUnits[sectionPos]
recordCandidates := func(unit *WorkUnitOutcome, enriched []proposals.EnrichedCorrectionProposal) {
unit.ProposalIndexOffset = nextProposalIndex
for _, candidate := range enriched {
candidate.ProposalIndex += nextProposalIndex
out.Candidates = append(out.Candidates, candidate)
if evidence.candidates {
for _, candidate := range enriched {
candidate.ProposalIndex += nextProposalIndex
out.Candidates = append(out.Candidates, candidate)
}
}
out.ProposalCount += len(enriched)
for operationIndex := range unit.Operations {
operation := &unit.Operations[operationIndex]
if operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded {
if evidence.reports && operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded {
for _, proposal := range enriched {
operation.ProposalIndices = append(operation.ProposalIndices, nextProposalIndex+proposal.ProposalIndex)
}
@@ -516,16 +555,17 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
interactions = capture
}
validationInput := validateSectionCandidatesInput{
Spec: input.Spec,
Policy: input.Policy,
WorkingTranscript: transcriptFromSection(sectionResult.section),
SectionIndex: &sectionMetadata.Index,
Validators: validatorsOrdered,
SectionEnriched: enriched,
Exchange: sectionResult.proposal.Exchange,
ValidationLLMClient: input.ValidationClient,
GlossaryView: input.GlossaryView,
Interactions: interactions,
Spec: input.Spec,
SuppressDecisionHistory: !evidence.decisions,
Policy: input.Policy,
WorkingTranscript: transcriptFromSection(sectionResult.section),
SectionIndex: &sectionMetadata.Index,
Validators: validatorsOrdered,
SectionEnriched: enriched,
Exchange: sectionResult.proposal.Exchange,
ValidationLLMClient: input.ValidationClient,
GlossaryView: input.GlossaryView,
Interactions: interactions,
}
if input.Spec.Settings.LLM.CorrectionRepairAttempts > 0 && len(enriched) > 0 {
validationInput.Section = validators.NewSectionLookup(validationInput.WorkingTranscript)
@@ -555,8 +595,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
failure: failure,
}
}
recordValidation := func(res sectionValidationResult) {
unit := &out.WorkUnits[res.sectionPos]
recordValidation := func(unit *WorkUnitOutcome, res sectionValidationResult) {
unit.Operations = append(unit.Operations, res.operations...)
unit.Repairs = res.repairs
mergeFilteredProposals(unit, res.filtered)
@@ -567,18 +606,30 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
unit.State = outcomes.WorkInterrupted
}
out.Approved = append(out.Approved, res.approved...)
out.ValidatorDecisions = append(out.ValidatorDecisions, res.decisions...)
out.ValidatorRejected = append(out.ValidatorRejected, res.rejected...)
if evidence.decisions {
out.ValidatorDecisions = append(out.ValidatorDecisions, res.decisions...)
out.ValidatorRejected = append(out.ValidatorRejected, res.rejected...)
}
if len(res.approved) > 0 {
out.rememberProposalRecovery(unit)
}
}
nextAdmission, active := 0, 0
for {
for work := pending[nextSectionToRecord]; work != nil && work.finished; work = pending[nextSectionToRecord] {
offset := nextProposalIndex
recordCandidates(nextSectionToRecord, work.candidates)
recordCandidates(&work.unit, work.candidates)
if work.validation != nil {
work.validation.assignProposalIndices(offset)
recordValidation(*work.validation)
recordValidation(&work.unit, *work.validation)
}
out.Totals.Record(work.unit.State, work.unit.Operations)
if evidence.reports {
out.WorkUnits[nextSectionToRecord] = work.unit
} else if evidence.candidates && len(work.unit.Repairs) > 0 {
work.unit.Operations = nil
out.WorkUnits = append(out.WorkUnits, work.unit)
}
delete(pending, nextSectionToRecord)
nextSectionToRecord++
@@ -586,7 +637,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
for len(pending) < moduleWorkWindow && nextAdmission < len(input.Sections) && runCtx.Err() == nil {
sectionPos := nextAdmission
nextAdmission++
pending[sectionPos] = &sectionWork{}
pending[sectionPos] = &sectionWork{unit: newWorkUnit(input.Spec, contracts.SectionMetadataFromSection(input.Sections[sectionPos]), 0)}
active++
workers.Go(func() {
proposed := proposeSection(runCtx, input, sectionPos)
@@ -600,13 +651,13 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
select {
case result := <-proposalResults:
active--
unit := &out.WorkUnits[result.sectionPos]
work := pending[result.sectionPos]
unit := &work.unit
unit.Operations = append(unit.Operations, result.proposal.Operations...)
if result.started {
unit.State = outcomes.WorkInterrupted
}
unit.Failure = result.failure
work := pending[result.sectionPos]
if result.started && result.failure == nil {
work.candidates = makeCandidates(result)
if runCtx.Err() == nil {
@@ -633,10 +684,6 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
return out.Approved[i].ProposalIndex < out.Approved[j].ProposalIndex
})
for _, unit := range out.WorkUnits {
out.Operations = append(out.Operations, unit.Operations...)
}
sort.SliceStable(out.ValidatorDecisions, func(i, j int) bool {
if out.ValidatorDecisions[i].ProposalIndex != out.ValidatorDecisions[j].ProposalIndex {
return out.ValidatorDecisions[i].ProposalIndex < out.ValidatorDecisions[j].ProposalIndex
@@ -668,19 +715,20 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
}
type validateSectionCandidatesInput struct {
Section *validators.SectionLookup
RepairProposalIndex *int
RepairAttempt *int
Exchange *structuredoutput.Exchange
Spec contracts.ModuleRunSpec
Policy proposals.ReplacementPolicy
WorkingTranscript *schema.Transcript
SectionIndex *int
Validators []validators.Validator
SectionEnriched []proposals.EnrichedCorrectionProposal
ValidationLLMClient contracts.StructuredLLMClient
Interactions structuredoutput.InteractionSink
GlossaryView *validators.GlossaryView
SuppressDecisionHistory bool
Section *validators.SectionLookup
RepairProposalIndex *int
RepairAttempt *int
Exchange *structuredoutput.Exchange
Spec contracts.ModuleRunSpec
Policy proposals.ReplacementPolicy
WorkingTranscript *schema.Transcript
SectionIndex *int
Validators []validators.Validator
SectionEnriched []proposals.EnrichedCorrectionProposal
ValidationLLMClient contracts.StructuredLLMClient
Interactions structuredoutput.InteractionSink
GlossaryView *validators.GlossaryView
}
type validateSectionCandidatesResult struct {
@@ -781,16 +829,18 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
noEffect[d.ProposalIndex] = true
}
}
decisions = append(decisions, ValidatorDecisionRecord{
ValidatorName: validator.Name(),
ExecutionClass: class,
ProposalIndex: d.ProposalIndex,
Approved: approved,
Disposition: disposition,
ReasonCode: d.ReasonCode,
Message: d.Message,
DiagnosticArtifactPath: d.DiagnosticArtifactPath,
})
if !input.SuppressDecisionHistory {
decisions = append(decisions, ValidatorDecisionRecord{
ValidatorName: validator.Name(),
ExecutionClass: class,
ProposalIndex: d.ProposalIndex,
Approved: approved,
Disposition: disposition,
ReasonCode: d.ReasonCode,
Message: d.Message,
DiagnosticArtifactPath: d.DiagnosticArtifactPath,
})
}
if approved {
nextEligible = append(nextEligible, byIndex[d.ProposalIndex])
continue
@@ -800,19 +850,21 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
filtered = append(filtered, outcomes.FilteredProposal{ProposalIndex: d.ProposalIndex, Reason: reason, Count: 1})
}
}
p := byIndex[d.ProposalIndex]
rejected = append(rejected, ValidatorRejectedChange{
ValidatorName: validator.Name(),
ProposalIndex: p.ProposalIndex,
ModuleKey: p.ModuleKey,
ModuleInstance: p.ModuleInstance,
TargetSegmentID: p.TargetSegmentID,
OriginalText: p.OriginalText,
CorrectedText: p.CorrectedText,
Disposition: disposition,
ReasonCode: d.ReasonCode,
Message: d.Message,
})
if feedback || !input.SuppressDecisionHistory {
p := byIndex[d.ProposalIndex]
rejected = append(rejected, ValidatorRejectedChange{
ValidatorName: validator.Name(),
ProposalIndex: p.ProposalIndex,
ModuleKey: p.ModuleKey,
ModuleInstance: p.ModuleInstance,
TargetSegmentID: p.TargetSegmentID,
OriginalText: p.OriginalText,
CorrectedText: p.CorrectedText,
Disposition: disposition,
ReasonCode: d.ReasonCode,
Message: d.Message,
})
}
}
if err != nil {
@@ -877,9 +929,9 @@ func mergeOperationFiltering(operation *outcomes.Operation, filtered []outcomes.
}
}
func attributeApplyFiltering(sections []chunking.Section, approved []proposals.EnrichedCorrectionProposal, skipped []proposals.SkippedChange, policy proposals.ReplacementPolicy, units []WorkUnitOutcome) {
func findApplyFiltering(sections []chunking.Section, approved []proposals.EnrichedCorrectionProposal, skipped []proposals.SkippedChange, policy proposals.ReplacementPolicy) map[int][]outcomes.FilteredProposal {
if len(skipped) == 0 {
return
return nil
}
byIndex := make(map[int]proposals.EnrichedCorrectionProposal, len(approved))
sectionPosition := make(map[int]int, len(sections))
@@ -916,26 +968,7 @@ func attributeApplyFiltering(sections []chunking.Section, approved []proposals.E
}
filtered[pos] = append(filtered[pos], outcomes.FilteredProposal{ProposalIndex: skip.ProposalIndex, Reason: reason, Count: 1})
}
for pos, items := range filtered {
unit := &units[pos]
revisions := make(map[int]*outcomes.Operation)
for i := range unit.Operations {
op := &unit.Operations[i]
if op.Reference.Kind == outcomes.OperationCorrectionRepair && op.Succeeded && op.Reference.ProposalIndex != nil &&
slices.Contains(op.RecoveryReasons, outcomes.RecoveryCorrectionRevised) {
revisions[*op.Reference.ProposalIndex] = op
}
}
original := items[:0]
for _, item := range items {
if op := revisions[item.ProposalIndex]; op != nil {
mergeOperationFiltering(op, []outcomes.FilteredProposal{item})
} else {
original = append(original, item)
}
}
mergeFilteredProposals(unit, original)
}
return filtered
}
func reorderValidatorsForPipeline(in []validators.Validator) ([]validators.Validator, map[string]int) {

View File

@@ -173,7 +173,7 @@ func TestCandidateCaptureIsExplicitAndIncludesFilteredProposals(t *testing.T) {
if len(without.Candidates) != 0 {
t.Fatalf("debug-off candidate capture: %+v", without.Candidates)
}
input.CaptureCandidates = true
input.Evidence = &EvidenceSelection{Reports: true, Debug: true}
with, err := r.Run(context.Background(), input)
if err != nil {
t.Fatal(err)
@@ -844,7 +844,7 @@ func TestRunnerApplyConflictAfterTwoInitiallyValidDeletions(t *testing.T) {
},
},
}})
out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, CaptureCandidates: true})
out, err := r.Run(context.Background(), RunInput{Transcript: transcript, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, Evidence: &EvidenceSelection{Reports: true, Debug: true}})
if err != nil {
t.Fatal(err)
}
@@ -973,8 +973,8 @@ func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress(
if len(out.ModuleResults) != 2 || len(out.ModuleResults[1].ValidatorRejected) != 1 {
t.Fatalf("expected second module rejection, got %+v", out.ModuleResults)
}
if len(out.ModuleResults[1].Operations) != 1 || out.ModuleResults[1].Operations[0].FailureReason != outcomes.ReasonValidationDecisionsInvalid {
t.Fatalf("expected invalid decision-set operation, got %+v", out.ModuleResults[1].Operations)
if len(out.WorkUnits[1].Operations) != 1 || out.WorkUnits[1].Operations[0].FailureReason != outcomes.ReasonValidationDecisionsInvalid {
t.Fatalf("expected invalid decision-set operation, got %+v", out.WorkUnits[1].Operations)
}
}

View File

@@ -218,7 +218,7 @@ func TestLaterRepairArtifactsUseOrderedProposalIndices(t *testing.T) {
cfg.Chunking.MaxSectionTokens = 1
cfg.Chunking.MinSectionTokens = 0
out, err := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}}).Run(ctx, RunInput{
Config: &cfg, Interactions: llm.NewDiagnosticsWriter(t.TempDir()), CaptureCandidates: true,
Config: &cfg, Interactions: llm.NewDiagnosticsWriter(t.TempDir()), Evidence: &EvidenceSelection{Reports: true, Debug: true},
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "alpha"}, {ID: 93, Text: "bravo"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{
ModuleKey: "grammar", InstanceName: "grammar", ProposalClient: proposal, ValidationClient: validation,

View File

@@ -0,0 +1,78 @@
package runner
import (
"slices"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
)
func newWorkUnit(spec contracts.ModuleRunSpec, section contracts.SectionMetadata, moduleIndex int) WorkUnitOutcome {
return WorkUnitOutcome{
ModuleIndex: moduleIndex, StageIndex: spec.Index, StageID: spec.ID, ModuleKey: spec.ModuleKey,
ModuleInstance: spec.InstanceName, Section: section, State: outcomes.WorkNotStarted,
}
}
type proposalRecovery struct {
recovered bool
filtered bool
}
// Application can discover additional defects. Keep only the proposal operation
// flags needed to update totals and the identities of approved revisions.
func (p *modulePipelineResult) rememberProposalRecovery(unit *WorkUnitOutcome) {
for _, operation := range unit.Operations {
if operation.Reference.Kind == outcomes.OperationProposal && operation.Succeeded {
if p.proposalFacts == nil {
p.proposalFacts = make(map[int]proposalRecovery)
}
p.proposalFacts[unit.Section.Index] = proposalRecovery{
recovered: len(operation.RecoveryReasons) > 0, filtered: len(operation.Filtered) > 0,
}
break
}
}
for _, repair := range unit.Repairs {
if repair.Outcome == "approved" {
if p.revised == nil {
p.revised = make(map[int]struct{})
}
p.revised[repair.ProposalIndex] = struct{}{}
}
}
}
func (p *modulePipelineResult) recordApplyFiltering(filters map[int][]outcomes.FilteredProposal, reports bool) {
for pos, items := range filters {
original := items[:0]
for _, item := range items {
if _, repaired := p.revised[item.ProposalIndex]; !repaired {
original = append(original, item)
continue
}
if reports {
operations := p.WorkUnits[pos].Operations
for i := range operations {
op := &operations[i]
if op.Reference.Kind == outcomes.OperationCorrectionRepair && op.Reference.ProposalIndex != nil &&
*op.Reference.ProposalIndex == item.ProposalIndex && slices.Contains(op.RecoveryReasons, outcomes.RecoveryCorrectionRevised) {
mergeOperationFiltering(op, []outcomes.FilteredProposal{item})
break
}
}
}
}
if facts, ok := p.proposalFacts[pos]; ok && len(original) != 0 {
if !facts.recovered {
p.Totals.Recoveries++
}
if !facts.filtered {
p.Totals.FilteringResponses++
}
if reports {
mergeFilteredProposals(&p.WorkUnits[pos], original)
}
}
}
}