Carry one proposal result from generation to the runner
This commit is contained in:
@@ -343,10 +343,7 @@ type collectSectionProposalsInput struct {
|
||||
type sectionProposalResult struct {
|
||||
sectionPos int
|
||||
section chunking.Section
|
||||
corrected []proposals.CorrectionProposal
|
||||
references []int
|
||||
exchange *structuredoutput.Exchange
|
||||
operations []outcomes.Operation
|
||||
proposal contracts.ProposalResult
|
||||
started bool
|
||||
err error
|
||||
}
|
||||
@@ -413,10 +410,7 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI
|
||||
results <- sectionProposalResult{
|
||||
sectionPos: sectionPos,
|
||||
section: section,
|
||||
corrected: proposalResult.Proposals,
|
||||
references: proposalResult.SegmentReferences,
|
||||
exchange: proposalResult.Exchange,
|
||||
operations: proposalResult.Operations,
|
||||
proposal: proposalResult,
|
||||
started: true,
|
||||
err: err,
|
||||
}
|
||||
@@ -486,7 +480,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
||||
if result.started {
|
||||
unit.State = outcomes.WorkInterrupted
|
||||
}
|
||||
unit.Operations = append(unit.Operations, result.operations...)
|
||||
unit.Operations = append(unit.Operations, result.proposal.Operations...)
|
||||
if result.err != nil {
|
||||
setErr(result.err)
|
||||
continue
|
||||
@@ -509,8 +503,8 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
||||
}
|
||||
delete(pending, nextSectionToProcess)
|
||||
sectionMeta := contracts.SectionMetadataFromSection(sectionResult.section)
|
||||
sectionEnriched := make([]proposals.EnrichedCorrectionProposal, 0, len(sectionResult.corrected))
|
||||
for i, corrected := range sectionResult.corrected {
|
||||
sectionEnriched := make([]proposals.EnrichedCorrectionProposal, 0, len(sectionResult.proposal.Proposals))
|
||||
for i, corrected := range sectionResult.proposal.Proposals {
|
||||
sectionIndex := sectionMeta.Index
|
||||
sectionEnriched = append(sectionEnriched, proposals.EnrichedCorrectionProposal{
|
||||
CorrectionProposal: corrected,
|
||||
@@ -544,7 +538,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
||||
SectionIndex: §ionMetadata.Index,
|
||||
Validators: validatorsOrdered,
|
||||
SectionEnriched: enriched,
|
||||
Exchange: sectionResult.exchange,
|
||||
Exchange: sectionResult.proposal.Exchange,
|
||||
ValidationLLMClient: input.ValidationClient,
|
||||
GlossaryView: input.GlossaryView,
|
||||
Interactions: input.Interactions,
|
||||
@@ -559,7 +553,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
|
||||
if provider, ok := input.Module.(interface{ RepairPromptID() string }); ok {
|
||||
promptID = provider.RepairPromptID()
|
||||
}
|
||||
validated, repairs, err = repairCandidates(runCtx, validationInput, sectionResult.references, validated, promptID)
|
||||
validated, repairs, err = repairCandidates(runCtx, validationInput, sectionResult.proposal.SegmentReferences, validated, promptID)
|
||||
}
|
||||
setErr(err)
|
||||
validationResults <- sectionValidationResult{
|
||||
|
||||
@@ -20,7 +20,9 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/modules/grammar"
|
||||
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
|
||||
)
|
||||
|
||||
@@ -449,86 +451,6 @@ func TestRunnerProposalsExecutePerChunkSection(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerProposalIndexOrderingIsDeterministicAcrossParallelSections(t *testing.T) {
|
||||
cfg := runnerTestConfig()
|
||||
cfg.Chunking.MaxSectionTokens = 3
|
||||
cfg.Chunking.MinSectionTokens = 0
|
||||
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{
|
||||
{ID: 1, Text: "alpha one"},
|
||||
{ID: 2, Text: "bravo two"},
|
||||
{ID: 3, Text: "charlie three"},
|
||||
}}
|
||||
|
||||
started := make(chan int, len(transcript.Segments))
|
||||
releaseByID := map[int]chan struct{}{
|
||||
1: make(chan struct{}),
|
||||
2: make(chan struct{}),
|
||||
3: make(chan struct{}),
|
||||
}
|
||||
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
if len(req.WorkingTranscript.Segments) != 1 {
|
||||
t.Fatalf("expected one segment per section, got %d", len(req.WorkingTranscript.Segments))
|
||||
}
|
||||
seg := req.WorkingTranscript.Segments[0]
|
||||
started <- seg.ID
|
||||
<-releaseByID[seg.ID]
|
||||
return []proposals.CorrectionProposal{
|
||||
{TargetSegmentID: seg.ID, OriginalText: seg.Text, CorrectedText: strings.ToUpper(seg.Text)},
|
||||
}, nil
|
||||
}},
|
||||
}})
|
||||
|
||||
resultCh := make(chan struct {
|
||||
out RunOutput
|
||||
err error
|
||||
}, 1)
|
||||
go func() {
|
||||
out, err := r.Run(context.Background(), RunInput{
|
||||
Config: &cfg,
|
||||
Transcript: transcript,
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{
|
||||
{ModuleKey: "m", InstanceName: "m"},
|
||||
},
|
||||
})
|
||||
resultCh <- struct {
|
||||
out RunOutput
|
||||
err error
|
||||
}{out: out, err: err}
|
||||
}()
|
||||
|
||||
waitForRunnerSectionIDs(t, started, map[int]struct{}{1: {}, 2: {}, 3: {}})
|
||||
close(releaseByID[3])
|
||||
close(releaseByID[2])
|
||||
close(releaseByID[1])
|
||||
|
||||
result := <-resultCh
|
||||
out, err := result.out, result.err
|
||||
if err != nil {
|
||||
t.Fatalf("Run error: %v", err)
|
||||
}
|
||||
if len(out.ModuleResults) != 1 {
|
||||
t.Fatalf("expected one module result, got %+v", out.ModuleResults)
|
||||
}
|
||||
if got := len(out.ModuleResults[0].AppliedChanges); got != 3 {
|
||||
t.Fatalf("expected three applied changes, got %d", got)
|
||||
}
|
||||
for i, change := range out.ModuleResults[0].AppliedChanges {
|
||||
wantID := i + 1
|
||||
if change.TargetSegmentID != wantID {
|
||||
t.Fatalf("expected deterministic applied-change order by section/segment, got %+v", out.ModuleResults[0].AppliedChanges)
|
||||
}
|
||||
}
|
||||
wantFinal := []string{"ALPHA ONE", "BRAVO TWO", "CHARLIE THREE"}
|
||||
for i, seg := range out.FinalTranscript.Segments {
|
||||
if seg.Text != wantFinal[i] {
|
||||
t.Fatalf("expected deterministic final transcript regardless of section completion order; got %+v", out.FinalTranscript.Segments)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func waitForRunnerEntries(t *testing.T, entered <-chan struct{}, want int, label string) {
|
||||
t.Helper()
|
||||
deadline := time.After(350 * time.Millisecond)
|
||||
@@ -1228,90 +1150,117 @@ func TestRunnerLLMValidatorDiagnosticsWritten(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
type proposalGenerationModule struct {
|
||||
key string
|
||||
policy proposals.ReplacementPolicy
|
||||
type proposalClientFunc func(context.Context, contracts.StructuredCompletionRequest, any) (contracts.StructuredCompletionResponse, error)
|
||||
|
||||
func (f proposalClientFunc) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
return f(ctx, req, out)
|
||||
}
|
||||
|
||||
func (m proposalGenerationModule) Key() string { return m.key }
|
||||
func (m proposalGenerationModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy }
|
||||
func (m proposalGenerationModule) Validators() []validators.Validator { return nil }
|
||||
func (m proposalGenerationModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
result, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
|
||||
ModuleKey: req.RunSpec.ModuleKey,
|
||||
ModuleInstance: req.RunSpec.InstanceName,
|
||||
ReplacementPolicy: req.RunSpec.ReplacementPolicy,
|
||||
WorkingTranscript: req.WorkingTranscript,
|
||||
PromptID: "modules.grammar.proposal",
|
||||
Variables: map[string]string{"TranscriptDescriptionBlock": "", "GlossaryJSON": "[]", "SectionJSON": "[]"},
|
||||
LLMClient: req.LLMClient,
|
||||
Interactions: req.Interactions,
|
||||
})
|
||||
if err != nil {
|
||||
return contracts.ProposalResult{}, err
|
||||
}
|
||||
return contracts.ProposalResult{Proposals: result.Corrections, Operations: result.Operations}, nil
|
||||
}
|
||||
|
||||
type fakeProposalStructuredClient struct {
|
||||
responses []proposal_generation.StructuredCorrectionSet
|
||||
}
|
||||
|
||||
func (f *fakeProposalStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
_ = ctx
|
||||
_ = req
|
||||
if len(f.responses) == 0 {
|
||||
return contracts.StructuredCompletionResponse{}, errors.New("unexpected call")
|
||||
}
|
||||
target, ok := out.(*proposal_generation.StructuredCorrectionSet)
|
||||
if !ok {
|
||||
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output type")
|
||||
}
|
||||
*target = f.responses[0]
|
||||
f.responses = f.responses[1:]
|
||||
return contracts.StructuredCompletionResponse{}, nil
|
||||
}
|
||||
|
||||
func TestRunnerProposalGenerationHelperFlowsThroughPipeline(t *testing.T) {
|
||||
client := &fakeProposalStructuredClient{
|
||||
responses: []proposal_generation.StructuredCorrectionSet{
|
||||
{
|
||||
Corrections: []proposal_generation.StructuredCorrectionProposal{
|
||||
{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
func TestRunnerProposalIndexOrderingIsDeterministicAcrossParallelSections(t *testing.T) {
|
||||
cfg := runnerTestConfig()
|
||||
diagDir := t.TempDir()
|
||||
cfg.Chunking.MaxSectionTokens = 3
|
||||
cfg.Chunking.MinSectionTokens = 0
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{
|
||||
{ID: 71, Text: "alpha one"},
|
||||
{ID: 903, Text: "bravo two"},
|
||||
}}
|
||||
responses := []proposal_generation.StructuredCorrectionSet{
|
||||
{Corrections: []proposal_generation.StructuredCorrectionProposal{
|
||||
{SegmentReference: 1, OriginalText: "one", CorrectedText: "ONE"},
|
||||
{SegmentReference: 1, OriginalText: "alpha", CorrectedText: "ALPHA"},
|
||||
}},
|
||||
{Corrections: []proposal_generation.StructuredCorrectionProposal{
|
||||
{SegmentReference: 1, OriginalText: "two", CorrectedText: "TWO"},
|
||||
}},
|
||||
}
|
||||
completedLaterSection := make(chan struct{})
|
||||
client := proposalClientFunc(func(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
if req.Context.SectionIndex == nil {
|
||||
return contracts.StructuredCompletionResponse{}, errors.New("missing section index")
|
||||
}
|
||||
index := *req.Context.SectionIndex
|
||||
if index < 0 || index >= len(responses) {
|
||||
return contracts.StructuredCompletionResponse{}, errors.New("unexpected section index")
|
||||
}
|
||||
if index == 0 {
|
||||
select {
|
||||
case <-completedLaterSection:
|
||||
case <-ctx.Done():
|
||||
return contracts.StructuredCompletionResponse{}, ctx.Err()
|
||||
}
|
||||
}
|
||||
*out.(*proposal_generation.StructuredCorrectionSet) = responses[index]
|
||||
return contracts.StructuredCompletionResponse{}, nil
|
||||
})
|
||||
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": proposalGenerationModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique},
|
||||
}})
|
||||
|
||||
out, err := r.Run(context.Background(), RunInput{
|
||||
Config: &cfg,
|
||||
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}},
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ProposalClient: client, Settings: config.Stage{}}},
|
||||
Interactions: llm.NewDiagnosticsWriter(diagDir),
|
||||
dir := t.TempDir()
|
||||
writer := llm.NewDiagnosticsWriter(dir)
|
||||
interactions := proposalOrderSink{writer: writer, laterResponse: completedLaterSection}
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": &grammar.Module{}}})
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
out, err := r.Run(ctx, RunInput{
|
||||
Config: &cfg, Transcript: transcript, Interactions: interactions,
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", ProposalClient: client}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected run error: %v", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out.FinalTranscript.Segments[0].Text != "the cat" {
|
||||
t.Fatalf("expected proposal-generated correction to apply, got %q", out.FinalTranscript.Segments[0].Text)
|
||||
want := []struct {
|
||||
target int
|
||||
original string
|
||||
}{{71, "one"}, {71, "alpha"}, {903, "two"}}
|
||||
changes := out.ModuleResults[0].AppliedChanges
|
||||
if len(changes) != len(want) {
|
||||
t.Fatalf("applied changes: %+v", changes)
|
||||
}
|
||||
if len(out.ModuleResults) != 1 || len(out.ModuleResults[0].AppliedChanges) != 1 {
|
||||
t.Fatalf("expected one applied change, got %+v", out.ModuleResults)
|
||||
for i, change := range changes {
|
||||
if change.ProposalIndex != i || change.TargetSegmentID != want[i].target || change.OriginalText != want[i].original {
|
||||
t.Fatalf("change %d lost response order or segment identity: %+v", i, change)
|
||||
}
|
||||
}
|
||||
matches, globErr := filepath.Glob(filepath.Join(diagDir, "m", "*proposal-generation*response-payload.json"))
|
||||
if globErr != nil {
|
||||
t.Fatalf("glob diagnostics: %v", globErr)
|
||||
wantIndices := [][]int{{0, 1}, {2}}
|
||||
if len(out.WorkUnits) != len(wantIndices) {
|
||||
t.Fatalf("work units: %+v", out.WorkUnits)
|
||||
}
|
||||
if len(matches) == 0 {
|
||||
t.Fatalf("expected proposal-generation diagnostics artifacts in %s", filepath.Join(diagDir, "m"))
|
||||
for i, unit := range out.WorkUnits {
|
||||
if len(unit.Operations) != 1 || !reflect.DeepEqual(unit.Operations[0].ProposalIndices, wantIndices[i]) {
|
||||
t.Fatalf("section %d lost proposal indices: %+v", i, unit.Operations)
|
||||
}
|
||||
}
|
||||
|
||||
paths, err := filepath.Glob(filepath.Join(dir, "grammar", "*response-payload.json"))
|
||||
if err != nil || len(paths) != len(responses) {
|
||||
t.Fatalf("proposal diagnostic paths=%v error=%v", paths, err)
|
||||
}
|
||||
for i, path := range paths {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var evidence struct {
|
||||
Decoded proposal_generation.StructuredCorrectionSet `json:"decoded"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &evidence); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(evidence.Decoded, responses[i]) {
|
||||
t.Fatalf("section %d lost original references: %+v", i, evidence.Decoded)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type proposalOrderSink struct {
|
||||
writer *llm.DiagnosticsWriter
|
||||
laterResponse chan struct{}
|
||||
}
|
||||
|
||||
func (s proposalOrderSink) WriteInteraction(interaction structuredoutput.Interaction) (structuredoutput.InteractionArtifacts, error) {
|
||||
artifacts, err := s.writer.WriteInteraction(interaction)
|
||||
if interaction.Context.SectionIndex != nil && *interaction.Context.SectionIndex == 1 {
|
||||
close(s.laterResponse)
|
||||
}
|
||||
return artifacts, err
|
||||
}
|
||||
|
||||
func TestFatalPartialDecisionsRejectUnknownAndDuplicateIdentity(t *testing.T) {
|
||||
|
||||
Reference in New Issue
Block a user