Carry one proposal result from generation to the runner

This commit is contained in:
2026-11-16 14:30:57 +00:00
parent c2c8050bc8
commit a0c70e8dce
9 changed files with 153 additions and 314 deletions

View File

@@ -343,10 +343,7 @@ type collectSectionProposalsInput struct {
type sectionProposalResult struct {
sectionPos int
section chunking.Section
corrected []proposals.CorrectionProposal
references []int
exchange *structuredoutput.Exchange
operations []outcomes.Operation
proposal contracts.ProposalResult
started bool
err error
}
@@ -413,10 +410,7 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI
results <- sectionProposalResult{
sectionPos: sectionPos,
section: section,
corrected: proposalResult.Proposals,
references: proposalResult.SegmentReferences,
exchange: proposalResult.Exchange,
operations: proposalResult.Operations,
proposal: proposalResult,
started: true,
err: err,
}
@@ -486,7 +480,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
if result.started {
unit.State = outcomes.WorkInterrupted
}
unit.Operations = append(unit.Operations, result.operations...)
unit.Operations = append(unit.Operations, result.proposal.Operations...)
if result.err != nil {
setErr(result.err)
continue
@@ -509,8 +503,8 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
}
delete(pending, nextSectionToProcess)
sectionMeta := contracts.SectionMetadataFromSection(sectionResult.section)
sectionEnriched := make([]proposals.EnrichedCorrectionProposal, 0, len(sectionResult.corrected))
for i, corrected := range sectionResult.corrected {
sectionEnriched := make([]proposals.EnrichedCorrectionProposal, 0, len(sectionResult.proposal.Proposals))
for i, corrected := range sectionResult.proposal.Proposals {
sectionIndex := sectionMeta.Index
sectionEnriched = append(sectionEnriched, proposals.EnrichedCorrectionProposal{
CorrectionProposal: corrected,
@@ -544,7 +538,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
SectionIndex: &sectionMetadata.Index,
Validators: validatorsOrdered,
SectionEnriched: enriched,
Exchange: sectionResult.exchange,
Exchange: sectionResult.proposal.Exchange,
ValidationLLMClient: input.ValidationClient,
GlossaryView: input.GlossaryView,
Interactions: input.Interactions,
@@ -559,7 +553,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
if provider, ok := input.Module.(interface{ RepairPromptID() string }); ok {
promptID = provider.RepairPromptID()
}
validated, repairs, err = repairCandidates(runCtx, validationInput, sectionResult.references, validated, promptID)
validated, repairs, err = repairCandidates(runCtx, validationInput, sectionResult.proposal.SegmentReferences, validated, promptID)
}
setErr(err)
validationResults <- sectionValidationResult{

View File

@@ -20,7 +20,9 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
"gitea.maximumdirect.net/eric/audita/v3/internal/modules/grammar"
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
)
@@ -449,86 +451,6 @@ func TestRunnerProposalsExecutePerChunkSection(t *testing.T) {
}
}
func TestRunnerProposalIndexOrderingIsDeterministicAcrossParallelSections(t *testing.T) {
cfg := runnerTestConfig()
cfg.Chunking.MaxSectionTokens = 3
cfg.Chunking.MinSectionTokens = 0
transcript := &schema.Transcript{Segments: []schema.Segment{
{ID: 1, Text: "alpha one"},
{ID: 2, Text: "bravo two"},
{ID: 3, Text: "charlie three"},
}}
started := make(chan int, len(transcript.Segments))
releaseByID := map[int]chan struct{}{
1: make(chan struct{}),
2: make(chan struct{}),
3: make(chan struct{}),
}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
if len(req.WorkingTranscript.Segments) != 1 {
t.Fatalf("expected one segment per section, got %d", len(req.WorkingTranscript.Segments))
}
seg := req.WorkingTranscript.Segments[0]
started <- seg.ID
<-releaseByID[seg.ID]
return []proposals.CorrectionProposal{
{TargetSegmentID: seg.ID, OriginalText: seg.Text, CorrectedText: strings.ToUpper(seg.Text)},
}, nil
}},
}})
resultCh := make(chan struct {
out RunOutput
err error
}, 1)
go func() {
out, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: transcript,
ModuleSpecs: []contracts.ModuleRunSpec{
{ModuleKey: "m", InstanceName: "m"},
},
})
resultCh <- struct {
out RunOutput
err error
}{out: out, err: err}
}()
waitForRunnerSectionIDs(t, started, map[int]struct{}{1: {}, 2: {}, 3: {}})
close(releaseByID[3])
close(releaseByID[2])
close(releaseByID[1])
result := <-resultCh
out, err := result.out, result.err
if err != nil {
t.Fatalf("Run error: %v", err)
}
if len(out.ModuleResults) != 1 {
t.Fatalf("expected one module result, got %+v", out.ModuleResults)
}
if got := len(out.ModuleResults[0].AppliedChanges); got != 3 {
t.Fatalf("expected three applied changes, got %d", got)
}
for i, change := range out.ModuleResults[0].AppliedChanges {
wantID := i + 1
if change.TargetSegmentID != wantID {
t.Fatalf("expected deterministic applied-change order by section/segment, got %+v", out.ModuleResults[0].AppliedChanges)
}
}
wantFinal := []string{"ALPHA ONE", "BRAVO TWO", "CHARLIE THREE"}
for i, seg := range out.FinalTranscript.Segments {
if seg.Text != wantFinal[i] {
t.Fatalf("expected deterministic final transcript regardless of section completion order; got %+v", out.FinalTranscript.Segments)
}
}
}
func waitForRunnerEntries(t *testing.T, entered <-chan struct{}, want int, label string) {
t.Helper()
deadline := time.After(350 * time.Millisecond)
@@ -1228,90 +1150,117 @@ func TestRunnerLLMValidatorDiagnosticsWritten(t *testing.T) {
}
}
type proposalGenerationModule struct {
key string
policy proposals.ReplacementPolicy
type proposalClientFunc func(context.Context, contracts.StructuredCompletionRequest, any) (contracts.StructuredCompletionResponse, error)
func (f proposalClientFunc) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
return f(ctx, req, out)
}
func (m proposalGenerationModule) Key() string { return m.key }
func (m proposalGenerationModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy }
func (m proposalGenerationModule) Validators() []validators.Validator { return nil }
func (m proposalGenerationModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
result, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
ModuleKey: req.RunSpec.ModuleKey,
ModuleInstance: req.RunSpec.InstanceName,
ReplacementPolicy: req.RunSpec.ReplacementPolicy,
WorkingTranscript: req.WorkingTranscript,
PromptID: "modules.grammar.proposal",
Variables: map[string]string{"TranscriptDescriptionBlock": "", "GlossaryJSON": "[]", "SectionJSON": "[]"},
LLMClient: req.LLMClient,
Interactions: req.Interactions,
})
if err != nil {
return contracts.ProposalResult{}, err
}
return contracts.ProposalResult{Proposals: result.Corrections, Operations: result.Operations}, nil
}
type fakeProposalStructuredClient struct {
responses []proposal_generation.StructuredCorrectionSet
}
func (f *fakeProposalStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
_ = ctx
_ = req
if len(f.responses) == 0 {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected call")
}
target, ok := out.(*proposal_generation.StructuredCorrectionSet)
if !ok {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output type")
}
*target = f.responses[0]
f.responses = f.responses[1:]
return contracts.StructuredCompletionResponse{}, nil
}
func TestRunnerProposalGenerationHelperFlowsThroughPipeline(t *testing.T) {
client := &fakeProposalStructuredClient{
responses: []proposal_generation.StructuredCorrectionSet{
{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the"},
},
},
},
}
func TestRunnerProposalIndexOrderingIsDeterministicAcrossParallelSections(t *testing.T) {
cfg := runnerTestConfig()
diagDir := t.TempDir()
cfg.Chunking.MaxSectionTokens = 3
cfg.Chunking.MinSectionTokens = 0
transcript := &schema.Transcript{Segments: []schema.Segment{
{ID: 71, Text: "alpha one"},
{ID: 903, Text: "bravo two"},
}}
responses := []proposal_generation.StructuredCorrectionSet{
{Corrections: []proposal_generation.StructuredCorrectionProposal{
{SegmentReference: 1, OriginalText: "one", CorrectedText: "ONE"},
{SegmentReference: 1, OriginalText: "alpha", CorrectedText: "ALPHA"},
}},
{Corrections: []proposal_generation.StructuredCorrectionProposal{
{SegmentReference: 1, OriginalText: "two", CorrectedText: "TWO"},
}},
}
completedLaterSection := make(chan struct{})
client := proposalClientFunc(func(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
if req.Context.SectionIndex == nil {
return contracts.StructuredCompletionResponse{}, errors.New("missing section index")
}
index := *req.Context.SectionIndex
if index < 0 || index >= len(responses) {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected section index")
}
if index == 0 {
select {
case <-completedLaterSection:
case <-ctx.Done():
return contracts.StructuredCompletionResponse{}, ctx.Err()
}
}
*out.(*proposal_generation.StructuredCorrectionSet) = responses[index]
return contracts.StructuredCompletionResponse{}, nil
})
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": proposalGenerationModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique},
}})
out, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ProposalClient: client, Settings: config.Stage{}}},
Interactions: llm.NewDiagnosticsWriter(diagDir),
dir := t.TempDir()
writer := llm.NewDiagnosticsWriter(dir)
interactions := proposalOrderSink{writer: writer, laterResponse: completedLaterSection}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": &grammar.Module{}}})
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
out, err := r.Run(ctx, RunInput{
Config: &cfg, Transcript: transcript, Interactions: interactions,
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", ProposalClient: client}},
})
if err != nil {
t.Fatalf("unexpected run error: %v", err)
t.Fatal(err)
}
if out.FinalTranscript.Segments[0].Text != "the cat" {
t.Fatalf("expected proposal-generated correction to apply, got %q", out.FinalTranscript.Segments[0].Text)
want := []struct {
target int
original string
}{{71, "one"}, {71, "alpha"}, {903, "two"}}
changes := out.ModuleResults[0].AppliedChanges
if len(changes) != len(want) {
t.Fatalf("applied changes: %+v", changes)
}
if len(out.ModuleResults) != 1 || len(out.ModuleResults[0].AppliedChanges) != 1 {
t.Fatalf("expected one applied change, got %+v", out.ModuleResults)
for i, change := range changes {
if change.ProposalIndex != i || change.TargetSegmentID != want[i].target || change.OriginalText != want[i].original {
t.Fatalf("change %d lost response order or segment identity: %+v", i, change)
}
}
matches, globErr := filepath.Glob(filepath.Join(diagDir, "m", "*proposal-generation*response-payload.json"))
if globErr != nil {
t.Fatalf("glob diagnostics: %v", globErr)
wantIndices := [][]int{{0, 1}, {2}}
if len(out.WorkUnits) != len(wantIndices) {
t.Fatalf("work units: %+v", out.WorkUnits)
}
if len(matches) == 0 {
t.Fatalf("expected proposal-generation diagnostics artifacts in %s", filepath.Join(diagDir, "m"))
for i, unit := range out.WorkUnits {
if len(unit.Operations) != 1 || !reflect.DeepEqual(unit.Operations[0].ProposalIndices, wantIndices[i]) {
t.Fatalf("section %d lost proposal indices: %+v", i, unit.Operations)
}
}
paths, err := filepath.Glob(filepath.Join(dir, "grammar", "*response-payload.json"))
if err != nil || len(paths) != len(responses) {
t.Fatalf("proposal diagnostic paths=%v error=%v", paths, err)
}
for i, path := range paths {
raw, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
var evidence struct {
Decoded proposal_generation.StructuredCorrectionSet `json:"decoded"`
}
if err := json.Unmarshal(raw, &evidence); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(evidence.Decoded, responses[i]) {
t.Fatalf("section %d lost original references: %+v", i, evidence.Decoded)
}
}
}
type proposalOrderSink struct {
writer *llm.DiagnosticsWriter
laterResponse chan struct{}
}
func (s proposalOrderSink) WriteInteraction(interaction structuredoutput.Interaction) (structuredoutput.InteractionArtifacts, error) {
artifacts, err := s.writer.WriteInteraction(interaction)
if interaction.Context.SectionIndex != nil && *interaction.Context.SectionIndex == 1 {
close(s.laterResponse)
}
return artifacts, err
}
func TestFatalPartialDecisionsRejectUnknownAndDuplicateIdentity(t *testing.T) {