Carry one proposal result from generation to the runner

This commit is contained in:
2026-11-16 14:30:57 +00:00
parent c2c8050bc8
commit a0c70e8dce
9 changed files with 153 additions and 314 deletions

View File

@@ -73,10 +73,10 @@ func TestRepairedProposalStillFacesDomainChecks(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if calls != 2 || len(generated.Enriched) != 1 {
if calls != 2 || len(generated.Proposals) != 1 {
t.Fatalf("repair path: calls=%d, result=%+v", calls, generated)
}
input := validators.Request{ModuleKey: "grammar", Settings: config.Stage{}, CandidateProposal: generated.Enriched, WorkingTranscript: transcript}
input := validators.Request{ModuleKey: "grammar", Settings: config.Stage{}, CandidateProposal: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: generated.Proposals[0]}}, WorkingTranscript: transcript}
missing, err := (validators.OriginalTextPresenceValidator{}).Validate(context.Background(), input)
if err != nil {
t.Fatal(err)

View File

@@ -42,7 +42,6 @@ type Request struct {
PromptID string `json:"prompt_id"`
Variables map[string]string `json:"variables"`
StageName string `json:"stage_name,omitempty"`
StartIndex int `json:"start_index"`
CaptureExchange bool
LLMClient contracts.StructuredLLMClient
Interactions structuredoutput.InteractionSink
@@ -50,12 +49,8 @@ type Request struct {
// Result contains generated candidate proposals and optional diagnostics paths.
type Result struct {
SegmentReferences []int `json:"-"`
Exchange *structuredoutput.Exchange `json:"-"`
Corrections []proposals.CorrectionProposal `json:"corrections"`
Enriched []proposals.EnrichedCorrectionProposal `json:"enriched"`
Operations []outcomes.Operation `json:"operations,omitempty"`
Artifacts structuredoutput.InteractionArtifacts `json:"artifacts,omitempty"`
contracts.ProposalResult
Artifacts structuredoutput.InteractionArtifacts `json:"artifacts,omitempty"`
}
// GenerateCandidates executes one structured LLM call and deterministically maps
@@ -67,9 +62,6 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
if strings.TrimSpace(req.ModuleInstance) == "" {
return Result{}, fmt.Errorf("module instance must not be empty")
}
if req.StartIndex < 0 {
return Result{}, fmt.Errorf("start index must be non-negative")
}
if req.LLMClient == nil {
return Result{}, fmt.Errorf("structured LLM client is required")
}
@@ -115,7 +107,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
"module_instance": req.ModuleInstance,
"replacement_policy": req.ReplacementPolicy,
"section": req.Section,
"start_index": req.StartIndex,
"start_index": 0, // Position of the first item within this response.
"segment_ids": segmentIDs(req.WorkingTranscript),
"prompt": completion.Prompt,
}
@@ -129,7 +121,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
if callErr != nil {
structuredoutput.RecordFailure(&operation, callErr)
result := Result{Operations: []outcomes.Operation{operation}, Artifacts: artifacts}
result := Result{ProposalResult: contracts.ProposalResult{Operations: []outcomes.Operation{operation}}, Artifacts: artifacts}
if operation.FailureClass == outcomes.FailureRecoverable {
return result, nil
}
@@ -140,39 +132,23 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
references := make([]int, 0, len(response.Corrections))
corrections := make([]proposals.CorrectionProposal, 0, len(response.Corrections))
enriched := make([]proposals.EnrichedCorrectionProposal, 0, len(response.Corrections))
for i, raw := range response.Corrections {
for _, raw := range response.Corrections {
references = append(references, raw.SegmentReference)
// Unknown references remain invalid candidates for the existing domain validators.
candidate := proposals.CorrectionProposal{
corrections = append(corrections, proposals.CorrectionProposal{
TargetSegmentID: proposals.SegmentIDAtReference(req.WorkingTranscript, raw.SegmentReference),
OriginalText: raw.OriginalText,
CorrectedText: raw.CorrectedText,
}
corrections = append(corrections, candidate)
enrichedCandidate := proposals.EnrichedCorrectionProposal{
CorrectionProposal: candidate,
ProposalMetadata: proposals.ProposalMetadata{
ProposalIndex: req.StartIndex + i,
ModuleKey: req.ModuleKey,
ModuleInstance: req.ModuleInstance,
},
}
if req.Section != nil {
sectionIndex := req.Section.Index
enrichedCandidate.SectionIndex = &sectionIndex
}
enriched = append(enriched, enrichedCandidate)
})
}
return Result{
Corrections: corrections,
SegmentReferences: references,
Exchange: completion.Exchange,
Enriched: enriched,
Operations: []outcomes.Operation{operation},
Artifacts: artifacts,
ProposalResult: contracts.ProposalResult{
Proposals: corrections,
SegmentReferences: references,
Exchange: completion.Exchange,
Operations: []outcomes.Operation{operation},
},
Artifacts: artifacts,
}, nil
}

View File

@@ -6,7 +6,6 @@ import (
"errors"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
@@ -73,7 +72,6 @@ func defaultRequest(t *testing.T) Request {
PromptID: "modules.grammar.proposal",
Variables: map[string]string{"SectionJSON": "[]"},
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 7, Text: "teh"}, {ID: 25, Text: "other"}}},
StartIndex: 0,
}
}
@@ -90,24 +88,17 @@ func TestGenerateCandidatesSuccess(t *testing.T) {
section := contracts.SectionMetadata{Index: 2}
req := defaultRequest(t)
req.LLMClient = client
req.StartIndex = 10
req.Section = &section
got, err := GenerateCandidates(context.Background(), req)
if err != nil {
t.Fatalf("GenerateCandidates error: %v", err)
}
if len(got.Corrections) != 1 || len(got.Enriched) != 1 {
if len(got.Proposals) != 1 || len(got.SegmentReferences) != 1 || got.SegmentReferences[0] != 1 {
t.Fatalf("unexpected proposal lengths: %+v", got)
}
if got.Corrections[0].TargetSegmentID != 7 {
t.Fatalf("local reference not resolved: %+v", got.Corrections)
}
if got.Enriched[0].ProposalIndex != 10 {
t.Fatalf("expected proposal index 10, got %d", got.Enriched[0].ProposalIndex)
}
if got.Enriched[0].SectionIndex == nil || *got.Enriched[0].SectionIndex != 2 {
t.Fatalf("expected section index 2, got %v", got.Enriched[0].SectionIndex)
if got.Proposals[0].TargetSegmentID != 7 {
t.Fatalf("local reference not resolved: %+v", got.Proposals)
}
if len(got.Operations) != 1 || !got.Operations[0].Succeeded || got.Operations[0].Reference.SectionIndex != 2 || got.Operations[0].Reference.Kind != outcomes.OperationProposal {
t.Fatalf("proposal operation identity: %+v", got.Operations)
@@ -184,11 +175,11 @@ func TestGenerateCandidatesInvalidCorrectionIsPreservedForLaterValidation(t *tes
if err != nil {
t.Fatalf("expected invalid correction to survive generation, got %v", err)
}
if len(result.Corrections) != 1 {
if len(result.Proposals) != 1 {
t.Fatalf("expected one correction, got %+v", result)
}
if result.Corrections[0].TargetSegmentID != 0 {
t.Fatalf("unexpected preserved correction: %+v", result.Corrections[0])
if result.Proposals[0].TargetSegmentID != 0 {
t.Fatalf("unexpected preserved correction: %+v", result.Proposals[0])
}
raw, err := os.ReadFile(result.Artifacts.ResponsePayloadPath)
if err != nil {
@@ -218,7 +209,7 @@ func TestGenerateCandidatesMalformedStructuredOutputReturnsFailedOperation(t *te
if err != nil {
t.Fatalf("expected recoverable model operation, got %v", err)
}
if len(result.Corrections) != 0 || len(result.Enriched) != 0 {
if len(result.Proposals) != 0 {
t.Fatalf("expected no proposals on malformed response, got %+v", result)
}
if len(result.Operations) != 1 || result.Operations[0].Succeeded || result.Operations[0].FailureReason != outcomes.ReasonInvalidOutputExhausted {
@@ -255,87 +246,11 @@ func TestGenerateCandidatesValidEmptyResponseIsSuccessful(t *testing.T) {
req := defaultRequest(t)
req.LLMClient = client
result, err := GenerateCandidates(context.Background(), req)
if err != nil || len(result.Corrections) != 0 || len(result.Operations) != 1 || !result.Operations[0].Succeeded {
if err != nil || len(result.Proposals) != 0 || len(result.Operations) != 1 || !result.Operations[0].Succeeded {
t.Fatalf("valid empty response: %+v, %v", result, err)
}
}
func TestGenerateCandidatesDeterministicIndexAssignment(t *testing.T) {
baseResponse := StructuredCorrectionSet{
Corrections: []StructuredCorrectionProposal{
{SegmentReference: 1, OriginalText: "a", CorrectedText: "A"},
{SegmentReference: 2, OriginalText: "b", CorrectedText: "B"},
},
}
clientA := &fakeStructuredClient{responses: []StructuredCorrectionSet{baseResponse}}
clientB := &fakeStructuredClient{responses: []StructuredCorrectionSet{baseResponse}}
reqA := defaultRequest(t)
reqA.LLMClient = clientA
reqA.StartIndex = 3
first, err := GenerateCandidates(context.Background(), reqA)
if err != nil {
t.Fatalf("first generation failed: %v", err)
}
reqB := defaultRequest(t)
reqB.LLMClient = clientB
reqB.StartIndex = 3
second, err := GenerateCandidates(context.Background(), reqB)
if err != nil {
t.Fatalf("second generation failed: %v", err)
}
if !reflect.DeepEqual(first.Enriched, second.Enriched) {
t.Fatalf("expected stable enriched proposals\nfirst=%+v\nsecond=%+v", first.Enriched, second.Enriched)
}
}
func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) {
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "alpha", CorrectedText: "ALPHA"}}},
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "charlie", CorrectedText: "CHARLIE"}}},
},
}
section0 := contracts.SectionMetadata{Index: 0}
section1 := contracts.SectionMetadata{Index: 1}
req0 := defaultRequest(t)
req0.LLMClient = client
req0.Section = &section0
req0.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "alpha"}}}
req0.StartIndex = 0
part0, err := GenerateCandidates(context.Background(), req0)
if err != nil {
t.Fatalf("section 0 generation failed: %v", err)
}
req1 := defaultRequest(t)
req1.LLMClient = client
req1.Section = &section1
req1.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{{ID: 903, Text: "charlie"}}}
req1.StartIndex = len(part0.Enriched)
part1, err := GenerateCandidates(context.Background(), req1)
if err != nil {
t.Fatalf("section 1 generation failed: %v", err)
}
all := append(append([]proposals.EnrichedCorrectionProposal(nil), part0.Enriched...), part1.Enriched...)
if len(all) != 2 {
t.Fatalf("expected 2 proposals, got %d", len(all))
}
if all[0].TargetSegmentID != 71 || all[1].TargetSegmentID != 903 {
t.Fatalf("section references did not reset: %+v", all)
}
if all[0].ProposalIndex != 0 || all[1].ProposalIndex != 1 {
t.Fatalf("unexpected proposal indexes: %d, %d", all[0].ProposalIndex, all[1].ProposalIndex)
}
if all[0].SectionIndex == nil || *all[0].SectionIndex != 0 || all[1].SectionIndex == nil || *all[1].SectionIndex != 1 {
t.Fatalf("unexpected section metadata: %+v", all)
}
}
func TestGenerateCandidatesDiagnosticsWritten(t *testing.T) {
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
@@ -375,6 +290,9 @@ func TestGenerateCandidatesDiagnosticsWritten(t *testing.T) {
if err := json.Unmarshal(raw, &metadata); err != nil {
t.Fatalf("unmarshal metadata artifact: %v", err)
}
if start, ok := metadata["start_index"].(float64); !ok || start != 0 {
t.Fatalf("missing response-local origin: %s", raw)
}
schemaMap, ok := metadata["response_schema"].(map[string]any)
if !ok {
t.Fatalf("expected response_schema metadata in diagnostics, got %T", metadata["response_schema"])
@@ -405,7 +323,7 @@ func TestGenerateCandidatesClientError(t *testing.T) {
if err == nil || !strings.Contains(err.Error(), "completion failed") {
t.Fatalf("expected completion failure, got %v", err)
}
if len(got.Operations) != 1 || got.Operations[0].FailureReason != outcomes.ReasonInternalFailure || len(got.Corrections) != 0 {
if len(got.Operations) != 1 || got.Operations[0].FailureReason != outcomes.ReasonInternalFailure || len(got.Proposals) != 0 {
t.Fatalf("expected fatal call facts without proposals, got %+v", got)
}
matches, globErr := filepath.Glob(filepath.Join(dir, req.ModuleInstance, "*error-payload.json"))

View File

@@ -62,19 +62,9 @@ func ExecuteModuleProposal(ctx context.Context, req ModuleProposalRequest) (cont
PromptID: promptMetadata.PromptID,
Variables: variables,
StageName: stagename.ModuleProposal(req.ProposalRequest.RunSpec.InstanceName, sectionIndexPtr(req.ProposalRequest.Section)),
StartIndex: 0,
CaptureExchange: req.ProposalRequest.RunSpec.CaptureProposalExchange,
LLMClient: req.ProposalRequest.LLMClient,
Interactions: req.ProposalRequest.Interactions,
})
if err != nil {
return contracts.ProposalResult{Operations: generated.Operations}, err
}
return contracts.ProposalResult{
Proposals: generated.Corrections,
SegmentReferences: generated.SegmentReferences,
Exchange: generated.Exchange,
Operations: generated.Operations,
}, nil
return generated.ProposalResult, err
}

View File

@@ -343,10 +343,7 @@ type collectSectionProposalsInput struct {
type sectionProposalResult struct {
sectionPos int
section chunking.Section
corrected []proposals.CorrectionProposal
references []int
exchange *structuredoutput.Exchange
operations []outcomes.Operation
proposal contracts.ProposalResult
started bool
err error
}
@@ -413,10 +410,7 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI
results <- sectionProposalResult{
sectionPos: sectionPos,
section: section,
corrected: proposalResult.Proposals,
references: proposalResult.SegmentReferences,
exchange: proposalResult.Exchange,
operations: proposalResult.Operations,
proposal: proposalResult,
started: true,
err: err,
}
@@ -486,7 +480,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
if result.started {
unit.State = outcomes.WorkInterrupted
}
unit.Operations = append(unit.Operations, result.operations...)
unit.Operations = append(unit.Operations, result.proposal.Operations...)
if result.err != nil {
setErr(result.err)
continue
@@ -509,8 +503,8 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
}
delete(pending, nextSectionToProcess)
sectionMeta := contracts.SectionMetadataFromSection(sectionResult.section)
sectionEnriched := make([]proposals.EnrichedCorrectionProposal, 0, len(sectionResult.corrected))
for i, corrected := range sectionResult.corrected {
sectionEnriched := make([]proposals.EnrichedCorrectionProposal, 0, len(sectionResult.proposal.Proposals))
for i, corrected := range sectionResult.proposal.Proposals {
sectionIndex := sectionMeta.Index
sectionEnriched = append(sectionEnriched, proposals.EnrichedCorrectionProposal{
CorrectionProposal: corrected,
@@ -544,7 +538,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
SectionIndex: &sectionMetadata.Index,
Validators: validatorsOrdered,
SectionEnriched: enriched,
Exchange: sectionResult.exchange,
Exchange: sectionResult.proposal.Exchange,
ValidationLLMClient: input.ValidationClient,
GlossaryView: input.GlossaryView,
Interactions: input.Interactions,
@@ -559,7 +553,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
if provider, ok := input.Module.(interface{ RepairPromptID() string }); ok {
promptID = provider.RepairPromptID()
}
validated, repairs, err = repairCandidates(runCtx, validationInput, sectionResult.references, validated, promptID)
validated, repairs, err = repairCandidates(runCtx, validationInput, sectionResult.proposal.SegmentReferences, validated, promptID)
}
setErr(err)
validationResults <- sectionValidationResult{

View File

@@ -20,7 +20,9 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
"gitea.maximumdirect.net/eric/audita/v3/internal/modules/grammar"
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
)
@@ -449,86 +451,6 @@ func TestRunnerProposalsExecutePerChunkSection(t *testing.T) {
}
}
func TestRunnerProposalIndexOrderingIsDeterministicAcrossParallelSections(t *testing.T) {
cfg := runnerTestConfig()
cfg.Chunking.MaxSectionTokens = 3
cfg.Chunking.MinSectionTokens = 0
transcript := &schema.Transcript{Segments: []schema.Segment{
{ID: 1, Text: "alpha one"},
{ID: 2, Text: "bravo two"},
{ID: 3, Text: "charlie three"},
}}
started := make(chan int, len(transcript.Segments))
releaseByID := map[int]chan struct{}{
1: make(chan struct{}),
2: make(chan struct{}),
3: make(chan struct{}),
}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
if len(req.WorkingTranscript.Segments) != 1 {
t.Fatalf("expected one segment per section, got %d", len(req.WorkingTranscript.Segments))
}
seg := req.WorkingTranscript.Segments[0]
started <- seg.ID
<-releaseByID[seg.ID]
return []proposals.CorrectionProposal{
{TargetSegmentID: seg.ID, OriginalText: seg.Text, CorrectedText: strings.ToUpper(seg.Text)},
}, nil
}},
}})
resultCh := make(chan struct {
out RunOutput
err error
}, 1)
go func() {
out, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: transcript,
ModuleSpecs: []contracts.ModuleRunSpec{
{ModuleKey: "m", InstanceName: "m"},
},
})
resultCh <- struct {
out RunOutput
err error
}{out: out, err: err}
}()
waitForRunnerSectionIDs(t, started, map[int]struct{}{1: {}, 2: {}, 3: {}})
close(releaseByID[3])
close(releaseByID[2])
close(releaseByID[1])
result := <-resultCh
out, err := result.out, result.err
if err != nil {
t.Fatalf("Run error: %v", err)
}
if len(out.ModuleResults) != 1 {
t.Fatalf("expected one module result, got %+v", out.ModuleResults)
}
if got := len(out.ModuleResults[0].AppliedChanges); got != 3 {
t.Fatalf("expected three applied changes, got %d", got)
}
for i, change := range out.ModuleResults[0].AppliedChanges {
wantID := i + 1
if change.TargetSegmentID != wantID {
t.Fatalf("expected deterministic applied-change order by section/segment, got %+v", out.ModuleResults[0].AppliedChanges)
}
}
wantFinal := []string{"ALPHA ONE", "BRAVO TWO", "CHARLIE THREE"}
for i, seg := range out.FinalTranscript.Segments {
if seg.Text != wantFinal[i] {
t.Fatalf("expected deterministic final transcript regardless of section completion order; got %+v", out.FinalTranscript.Segments)
}
}
}
func waitForRunnerEntries(t *testing.T, entered <-chan struct{}, want int, label string) {
t.Helper()
deadline := time.After(350 * time.Millisecond)
@@ -1228,90 +1150,117 @@ func TestRunnerLLMValidatorDiagnosticsWritten(t *testing.T) {
}
}
type proposalGenerationModule struct {
key string
policy proposals.ReplacementPolicy
type proposalClientFunc func(context.Context, contracts.StructuredCompletionRequest, any) (contracts.StructuredCompletionResponse, error)
func (f proposalClientFunc) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
return f(ctx, req, out)
}
func (m proposalGenerationModule) Key() string { return m.key }
func (m proposalGenerationModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy }
func (m proposalGenerationModule) Validators() []validators.Validator { return nil }
func (m proposalGenerationModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
result, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
ModuleKey: req.RunSpec.ModuleKey,
ModuleInstance: req.RunSpec.InstanceName,
ReplacementPolicy: req.RunSpec.ReplacementPolicy,
WorkingTranscript: req.WorkingTranscript,
PromptID: "modules.grammar.proposal",
Variables: map[string]string{"TranscriptDescriptionBlock": "", "GlossaryJSON": "[]", "SectionJSON": "[]"},
LLMClient: req.LLMClient,
Interactions: req.Interactions,
})
if err != nil {
return contracts.ProposalResult{}, err
}
return contracts.ProposalResult{Proposals: result.Corrections, Operations: result.Operations}, nil
}
type fakeProposalStructuredClient struct {
responses []proposal_generation.StructuredCorrectionSet
}
func (f *fakeProposalStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
_ = ctx
_ = req
if len(f.responses) == 0 {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected call")
}
target, ok := out.(*proposal_generation.StructuredCorrectionSet)
if !ok {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output type")
}
*target = f.responses[0]
f.responses = f.responses[1:]
return contracts.StructuredCompletionResponse{}, nil
}
func TestRunnerProposalGenerationHelperFlowsThroughPipeline(t *testing.T) {
client := &fakeProposalStructuredClient{
responses: []proposal_generation.StructuredCorrectionSet{
{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the"},
},
},
},
}
func TestRunnerProposalIndexOrderingIsDeterministicAcrossParallelSections(t *testing.T) {
cfg := runnerTestConfig()
diagDir := t.TempDir()
cfg.Chunking.MaxSectionTokens = 3
cfg.Chunking.MinSectionTokens = 0
transcript := &schema.Transcript{Segments: []schema.Segment{
{ID: 71, Text: "alpha one"},
{ID: 903, Text: "bravo two"},
}}
responses := []proposal_generation.StructuredCorrectionSet{
{Corrections: []proposal_generation.StructuredCorrectionProposal{
{SegmentReference: 1, OriginalText: "one", CorrectedText: "ONE"},
{SegmentReference: 1, OriginalText: "alpha", CorrectedText: "ALPHA"},
}},
{Corrections: []proposal_generation.StructuredCorrectionProposal{
{SegmentReference: 1, OriginalText: "two", CorrectedText: "TWO"},
}},
}
completedLaterSection := make(chan struct{})
client := proposalClientFunc(func(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
if req.Context.SectionIndex == nil {
return contracts.StructuredCompletionResponse{}, errors.New("missing section index")
}
index := *req.Context.SectionIndex
if index < 0 || index >= len(responses) {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected section index")
}
if index == 0 {
select {
case <-completedLaterSection:
case <-ctx.Done():
return contracts.StructuredCompletionResponse{}, ctx.Err()
}
}
*out.(*proposal_generation.StructuredCorrectionSet) = responses[index]
return contracts.StructuredCompletionResponse{}, nil
})
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": proposalGenerationModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique},
}})
out, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ProposalClient: client, Settings: config.Stage{}}},
Interactions: llm.NewDiagnosticsWriter(diagDir),
dir := t.TempDir()
writer := llm.NewDiagnosticsWriter(dir)
interactions := proposalOrderSink{writer: writer, laterResponse: completedLaterSection}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": &grammar.Module{}}})
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
out, err := r.Run(ctx, RunInput{
Config: &cfg, Transcript: transcript, Interactions: interactions,
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", ProposalClient: client}},
})
if err != nil {
t.Fatalf("unexpected run error: %v", err)
t.Fatal(err)
}
if out.FinalTranscript.Segments[0].Text != "the cat" {
t.Fatalf("expected proposal-generated correction to apply, got %q", out.FinalTranscript.Segments[0].Text)
want := []struct {
target int
original string
}{{71, "one"}, {71, "alpha"}, {903, "two"}}
changes := out.ModuleResults[0].AppliedChanges
if len(changes) != len(want) {
t.Fatalf("applied changes: %+v", changes)
}
if len(out.ModuleResults) != 1 || len(out.ModuleResults[0].AppliedChanges) != 1 {
t.Fatalf("expected one applied change, got %+v", out.ModuleResults)
for i, change := range changes {
if change.ProposalIndex != i || change.TargetSegmentID != want[i].target || change.OriginalText != want[i].original {
t.Fatalf("change %d lost response order or segment identity: %+v", i, change)
}
}
matches, globErr := filepath.Glob(filepath.Join(diagDir, "m", "*proposal-generation*response-payload.json"))
if globErr != nil {
t.Fatalf("glob diagnostics: %v", globErr)
wantIndices := [][]int{{0, 1}, {2}}
if len(out.WorkUnits) != len(wantIndices) {
t.Fatalf("work units: %+v", out.WorkUnits)
}
if len(matches) == 0 {
t.Fatalf("expected proposal-generation diagnostics artifacts in %s", filepath.Join(diagDir, "m"))
for i, unit := range out.WorkUnits {
if len(unit.Operations) != 1 || !reflect.DeepEqual(unit.Operations[0].ProposalIndices, wantIndices[i]) {
t.Fatalf("section %d lost proposal indices: %+v", i, unit.Operations)
}
}
paths, err := filepath.Glob(filepath.Join(dir, "grammar", "*response-payload.json"))
if err != nil || len(paths) != len(responses) {
t.Fatalf("proposal diagnostic paths=%v error=%v", paths, err)
}
for i, path := range paths {
raw, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
var evidence struct {
Decoded proposal_generation.StructuredCorrectionSet `json:"decoded"`
}
if err := json.Unmarshal(raw, &evidence); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(evidence.Decoded, responses[i]) {
t.Fatalf("section %d lost original references: %+v", i, evidence.Decoded)
}
}
}
type proposalOrderSink struct {
writer *llm.DiagnosticsWriter
laterResponse chan struct{}
}
func (s proposalOrderSink) WriteInteraction(interaction structuredoutput.Interaction) (structuredoutput.InteractionArtifacts, error) {
artifacts, err := s.writer.WriteInteraction(interaction)
if interaction.Context.SectionIndex != nil && *interaction.Context.SectionIndex == 1 {
close(s.laterResponse)
}
return artifacts, err
}
func TestFatalPartialDecisionsRejectUnknownAndDuplicateIdentity(t *testing.T) {