Carry one proposal result from generation to the runner

This commit is contained in:
2026-11-16 14:30:57 +00:00
parent c2c8050bc8
commit a0c70e8dce
9 changed files with 153 additions and 314 deletions

View File

@@ -43,7 +43,9 @@ rendered by ConfigKit, the correction ledger, LLM execution observations, and
model interactions. Candidate capture includes proposals later
rejected or interrupted. Repair and revalidation artifacts use section, proposal,
attempt, validator, and batch coordinates as applicable. Request metadata records
these coordinates with the interaction role. Each interaction retains its available
these coordinates with the interaction role. Proposal request metadata carries
the zero-based response origin (`start_index`) and the ordered `segment_ids`
used to interpret response references. Each interaction retains its available
prepared request, response, and failure evidence. Ledger mapping preserves empty
proposed and applied replacements and distinguishes unavailable candidates from
interrupted work.

View File

@@ -13,8 +13,10 @@ formats; [configuration](../config.md) owns settings; the
3. The runner partitions the normalized transcript exactly once before its
module loop. Initial section identity, segment membership, and token
estimates stay fixed while text evolves.
4. Module instances execute in configured order. Sections may propose
concurrently, but proposal indices follow section order and response order.
4. Module instances execute in configured order. Each section hands its decoded
proposals and original local segment references to the runner along with
operation facts and any captured exchange. Sections may propose concurrently.
The runner assigns indices in section order and response order.
5. Deterministic validators precede LLM-backed validators. Every submitted
candidate receives one decision per validator on a normal return. Fatal returns
may preserve a checked subset of completed decisions as diagnostic evidence.

View File

@@ -472,6 +472,14 @@ Audit #46.
verify indices and original references. Keep model response and reference
coverage in the generator's owning tests.
**Complete.** The generator and runner share one decoded proposal result with
raw local references, operation facts, and the optional native exchange. The
runner assigns indices in section and response order. A production-module
fixture observed the later response first and verified applied indices,
operation associations, and raw references. Proposal diagnostics retain
the response-local `start_index`. The full offline suite and repeated
ordering checks under the race detector passed.
## Stage 14 — Own each correction repair branch explicitly
Audit #34, #45, #48, #64 (branch integration).

View File

@@ -73,10 +73,10 @@ func TestRepairedProposalStillFacesDomainChecks(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if calls != 2 || len(generated.Enriched) != 1 {
if calls != 2 || len(generated.Proposals) != 1 {
t.Fatalf("repair path: calls=%d, result=%+v", calls, generated)
}
input := validators.Request{ModuleKey: "grammar", Settings: config.Stage{}, CandidateProposal: generated.Enriched, WorkingTranscript: transcript}
input := validators.Request{ModuleKey: "grammar", Settings: config.Stage{}, CandidateProposal: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: generated.Proposals[0]}}, WorkingTranscript: transcript}
missing, err := (validators.OriginalTextPresenceValidator{}).Validate(context.Background(), input)
if err != nil {
t.Fatal(err)

View File

@@ -42,7 +42,6 @@ type Request struct {
PromptID string `json:"prompt_id"`
Variables map[string]string `json:"variables"`
StageName string `json:"stage_name,omitempty"`
StartIndex int `json:"start_index"`
CaptureExchange bool
LLMClient contracts.StructuredLLMClient
Interactions structuredoutput.InteractionSink
@@ -50,12 +49,8 @@ type Request struct {
// Result contains generated candidate proposals and optional diagnostics paths.
type Result struct {
SegmentReferences []int `json:"-"`
Exchange *structuredoutput.Exchange `json:"-"`
Corrections []proposals.CorrectionProposal `json:"corrections"`
Enriched []proposals.EnrichedCorrectionProposal `json:"enriched"`
Operations []outcomes.Operation `json:"operations,omitempty"`
Artifacts structuredoutput.InteractionArtifacts `json:"artifacts,omitempty"`
contracts.ProposalResult
Artifacts structuredoutput.InteractionArtifacts `json:"artifacts,omitempty"`
}
// GenerateCandidates executes one structured LLM call and deterministically maps
@@ -67,9 +62,6 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
if strings.TrimSpace(req.ModuleInstance) == "" {
return Result{}, fmt.Errorf("module instance must not be empty")
}
if req.StartIndex < 0 {
return Result{}, fmt.Errorf("start index must be non-negative")
}
if req.LLMClient == nil {
return Result{}, fmt.Errorf("structured LLM client is required")
}
@@ -115,7 +107,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
"module_instance": req.ModuleInstance,
"replacement_policy": req.ReplacementPolicy,
"section": req.Section,
"start_index": req.StartIndex,
"start_index": 0, // Position of the first item within this response.
"segment_ids": segmentIDs(req.WorkingTranscript),
"prompt": completion.Prompt,
}
@@ -129,7 +121,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
if callErr != nil {
structuredoutput.RecordFailure(&operation, callErr)
result := Result{Operations: []outcomes.Operation{operation}, Artifacts: artifacts}
result := Result{ProposalResult: contracts.ProposalResult{Operations: []outcomes.Operation{operation}}, Artifacts: artifacts}
if operation.FailureClass == outcomes.FailureRecoverable {
return result, nil
}
@@ -140,39 +132,23 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
references := make([]int, 0, len(response.Corrections))
corrections := make([]proposals.CorrectionProposal, 0, len(response.Corrections))
enriched := make([]proposals.EnrichedCorrectionProposal, 0, len(response.Corrections))
for i, raw := range response.Corrections {
for _, raw := range response.Corrections {
references = append(references, raw.SegmentReference)
// Unknown references remain invalid candidates for the existing domain validators.
candidate := proposals.CorrectionProposal{
corrections = append(corrections, proposals.CorrectionProposal{
TargetSegmentID: proposals.SegmentIDAtReference(req.WorkingTranscript, raw.SegmentReference),
OriginalText: raw.OriginalText,
CorrectedText: raw.CorrectedText,
}
corrections = append(corrections, candidate)
enrichedCandidate := proposals.EnrichedCorrectionProposal{
CorrectionProposal: candidate,
ProposalMetadata: proposals.ProposalMetadata{
ProposalIndex: req.StartIndex + i,
ModuleKey: req.ModuleKey,
ModuleInstance: req.ModuleInstance,
},
}
if req.Section != nil {
sectionIndex := req.Section.Index
enrichedCandidate.SectionIndex = &sectionIndex
}
enriched = append(enriched, enrichedCandidate)
})
}
return Result{
Corrections: corrections,
SegmentReferences: references,
Exchange: completion.Exchange,
Enriched: enriched,
Operations: []outcomes.Operation{operation},
Artifacts: artifacts,
ProposalResult: contracts.ProposalResult{
Proposals: corrections,
SegmentReferences: references,
Exchange: completion.Exchange,
Operations: []outcomes.Operation{operation},
},
Artifacts: artifacts,
}, nil
}

View File

@@ -6,7 +6,6 @@ import (
"errors"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
@@ -73,7 +72,6 @@ func defaultRequest(t *testing.T) Request {
PromptID: "modules.grammar.proposal",
Variables: map[string]string{"SectionJSON": "[]"},
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 7, Text: "teh"}, {ID: 25, Text: "other"}}},
StartIndex: 0,
}
}
@@ -90,24 +88,17 @@ func TestGenerateCandidatesSuccess(t *testing.T) {
section := contracts.SectionMetadata{Index: 2}
req := defaultRequest(t)
req.LLMClient = client
req.StartIndex = 10
req.Section = &section
got, err := GenerateCandidates(context.Background(), req)
if err != nil {
t.Fatalf("GenerateCandidates error: %v", err)
}
if len(got.Corrections) != 1 || len(got.Enriched) != 1 {
if len(got.Proposals) != 1 || len(got.SegmentReferences) != 1 || got.SegmentReferences[0] != 1 {
t.Fatalf("unexpected proposal lengths: %+v", got)
}
if got.Corrections[0].TargetSegmentID != 7 {
t.Fatalf("local reference not resolved: %+v", got.Corrections)
}
if got.Enriched[0].ProposalIndex != 10 {
t.Fatalf("expected proposal index 10, got %d", got.Enriched[0].ProposalIndex)
}
if got.Enriched[0].SectionIndex == nil || *got.Enriched[0].SectionIndex != 2 {
t.Fatalf("expected section index 2, got %v", got.Enriched[0].SectionIndex)
if got.Proposals[0].TargetSegmentID != 7 {
t.Fatalf("local reference not resolved: %+v", got.Proposals)
}
if len(got.Operations) != 1 || !got.Operations[0].Succeeded || got.Operations[0].Reference.SectionIndex != 2 || got.Operations[0].Reference.Kind != outcomes.OperationProposal {
t.Fatalf("proposal operation identity: %+v", got.Operations)
@@ -184,11 +175,11 @@ func TestGenerateCandidatesInvalidCorrectionIsPreservedForLaterValidation(t *tes
if err != nil {
t.Fatalf("expected invalid correction to survive generation, got %v", err)
}
if len(result.Corrections) != 1 {
if len(result.Proposals) != 1 {
t.Fatalf("expected one correction, got %+v", result)
}
if result.Corrections[0].TargetSegmentID != 0 {
t.Fatalf("unexpected preserved correction: %+v", result.Corrections[0])
if result.Proposals[0].TargetSegmentID != 0 {
t.Fatalf("unexpected preserved correction: %+v", result.Proposals[0])
}
raw, err := os.ReadFile(result.Artifacts.ResponsePayloadPath)
if err != nil {
@@ -218,7 +209,7 @@ func TestGenerateCandidatesMalformedStructuredOutputReturnsFailedOperation(t *te
if err != nil {
t.Fatalf("expected recoverable model operation, got %v", err)
}
if len(result.Corrections) != 0 || len(result.Enriched) != 0 {
if len(result.Proposals) != 0 {
t.Fatalf("expected no proposals on malformed response, got %+v", result)
}
if len(result.Operations) != 1 || result.Operations[0].Succeeded || result.Operations[0].FailureReason != outcomes.ReasonInvalidOutputExhausted {
@@ -255,87 +246,11 @@ func TestGenerateCandidatesValidEmptyResponseIsSuccessful(t *testing.T) {
req := defaultRequest(t)
req.LLMClient = client
result, err := GenerateCandidates(context.Background(), req)
if err != nil || len(result.Corrections) != 0 || len(result.Operations) != 1 || !result.Operations[0].Succeeded {
if err != nil || len(result.Proposals) != 0 || len(result.Operations) != 1 || !result.Operations[0].Succeeded {
t.Fatalf("valid empty response: %+v, %v", result, err)
}
}
func TestGenerateCandidatesDeterministicIndexAssignment(t *testing.T) {
baseResponse := StructuredCorrectionSet{
Corrections: []StructuredCorrectionProposal{
{SegmentReference: 1, OriginalText: "a", CorrectedText: "A"},
{SegmentReference: 2, OriginalText: "b", CorrectedText: "B"},
},
}
clientA := &fakeStructuredClient{responses: []StructuredCorrectionSet{baseResponse}}
clientB := &fakeStructuredClient{responses: []StructuredCorrectionSet{baseResponse}}
reqA := defaultRequest(t)
reqA.LLMClient = clientA
reqA.StartIndex = 3
first, err := GenerateCandidates(context.Background(), reqA)
if err != nil {
t.Fatalf("first generation failed: %v", err)
}
reqB := defaultRequest(t)
reqB.LLMClient = clientB
reqB.StartIndex = 3
second, err := GenerateCandidates(context.Background(), reqB)
if err != nil {
t.Fatalf("second generation failed: %v", err)
}
if !reflect.DeepEqual(first.Enriched, second.Enriched) {
t.Fatalf("expected stable enriched proposals\nfirst=%+v\nsecond=%+v", first.Enriched, second.Enriched)
}
}
func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) {
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "alpha", CorrectedText: "ALPHA"}}},
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "charlie", CorrectedText: "CHARLIE"}}},
},
}
section0 := contracts.SectionMetadata{Index: 0}
section1 := contracts.SectionMetadata{Index: 1}
req0 := defaultRequest(t)
req0.LLMClient = client
req0.Section = &section0
req0.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "alpha"}}}
req0.StartIndex = 0
part0, err := GenerateCandidates(context.Background(), req0)
if err != nil {
t.Fatalf("section 0 generation failed: %v", err)
}
req1 := defaultRequest(t)
req1.LLMClient = client
req1.Section = &section1
req1.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{{ID: 903, Text: "charlie"}}}
req1.StartIndex = len(part0.Enriched)
part1, err := GenerateCandidates(context.Background(), req1)
if err != nil {
t.Fatalf("section 1 generation failed: %v", err)
}
all := append(append([]proposals.EnrichedCorrectionProposal(nil), part0.Enriched...), part1.Enriched...)
if len(all) != 2 {
t.Fatalf("expected 2 proposals, got %d", len(all))
}
if all[0].TargetSegmentID != 71 || all[1].TargetSegmentID != 903 {
t.Fatalf("section references did not reset: %+v", all)
}
if all[0].ProposalIndex != 0 || all[1].ProposalIndex != 1 {
t.Fatalf("unexpected proposal indexes: %d, %d", all[0].ProposalIndex, all[1].ProposalIndex)
}
if all[0].SectionIndex == nil || *all[0].SectionIndex != 0 || all[1].SectionIndex == nil || *all[1].SectionIndex != 1 {
t.Fatalf("unexpected section metadata: %+v", all)
}
}
func TestGenerateCandidatesDiagnosticsWritten(t *testing.T) {
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
@@ -375,6 +290,9 @@ func TestGenerateCandidatesDiagnosticsWritten(t *testing.T) {
if err := json.Unmarshal(raw, &metadata); err != nil {
t.Fatalf("unmarshal metadata artifact: %v", err)
}
if start, ok := metadata["start_index"].(float64); !ok || start != 0 {
t.Fatalf("missing response-local origin: %s", raw)
}
schemaMap, ok := metadata["response_schema"].(map[string]any)
if !ok {
t.Fatalf("expected response_schema metadata in diagnostics, got %T", metadata["response_schema"])
@@ -405,7 +323,7 @@ func TestGenerateCandidatesClientError(t *testing.T) {
if err == nil || !strings.Contains(err.Error(), "completion failed") {
t.Fatalf("expected completion failure, got %v", err)
}
if len(got.Operations) != 1 || got.Operations[0].FailureReason != outcomes.ReasonInternalFailure || len(got.Corrections) != 0 {
if len(got.Operations) != 1 || got.Operations[0].FailureReason != outcomes.ReasonInternalFailure || len(got.Proposals) != 0 {
t.Fatalf("expected fatal call facts without proposals, got %+v", got)
}
matches, globErr := filepath.Glob(filepath.Join(dir, req.ModuleInstance, "*error-payload.json"))

View File

@@ -62,19 +62,9 @@ func ExecuteModuleProposal(ctx context.Context, req ModuleProposalRequest) (cont
PromptID: promptMetadata.PromptID,
Variables: variables,
StageName: stagename.ModuleProposal(req.ProposalRequest.RunSpec.InstanceName, sectionIndexPtr(req.ProposalRequest.Section)),
StartIndex: 0,
CaptureExchange: req.ProposalRequest.RunSpec.CaptureProposalExchange,
LLMClient: req.ProposalRequest.LLMClient,
Interactions: req.ProposalRequest.Interactions,
})
if err != nil {
return contracts.ProposalResult{Operations: generated.Operations}, err
}
return contracts.ProposalResult{
Proposals: generated.Corrections,
SegmentReferences: generated.SegmentReferences,
Exchange: generated.Exchange,
Operations: generated.Operations,
}, nil
return generated.ProposalResult, err
}

View File

@@ -343,10 +343,7 @@ type collectSectionProposalsInput struct {
type sectionProposalResult struct {
sectionPos int
section chunking.Section
corrected []proposals.CorrectionProposal
references []int
exchange *structuredoutput.Exchange
operations []outcomes.Operation
proposal contracts.ProposalResult
started bool
err error
}
@@ -413,10 +410,7 @@ func collectSectionProposals(ctx context.Context, input collectSectionProposalsI
results <- sectionProposalResult{
sectionPos: sectionPos,
section: section,
corrected: proposalResult.Proposals,
references: proposalResult.SegmentReferences,
exchange: proposalResult.Exchange,
operations: proposalResult.Operations,
proposal: proposalResult,
started: true,
err: err,
}
@@ -486,7 +480,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
if result.started {
unit.State = outcomes.WorkInterrupted
}
unit.Operations = append(unit.Operations, result.operations...)
unit.Operations = append(unit.Operations, result.proposal.Operations...)
if result.err != nil {
setErr(result.err)
continue
@@ -509,8 +503,8 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
}
delete(pending, nextSectionToProcess)
sectionMeta := contracts.SectionMetadataFromSection(sectionResult.section)
sectionEnriched := make([]proposals.EnrichedCorrectionProposal, 0, len(sectionResult.corrected))
for i, corrected := range sectionResult.corrected {
sectionEnriched := make([]proposals.EnrichedCorrectionProposal, 0, len(sectionResult.proposal.Proposals))
for i, corrected := range sectionResult.proposal.Proposals {
sectionIndex := sectionMeta.Index
sectionEnriched = append(sectionEnriched, proposals.EnrichedCorrectionProposal{
CorrectionProposal: corrected,
@@ -544,7 +538,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
SectionIndex: &sectionMetadata.Index,
Validators: validatorsOrdered,
SectionEnriched: enriched,
Exchange: sectionResult.exchange,
Exchange: sectionResult.proposal.Exchange,
ValidationLLMClient: input.ValidationClient,
GlossaryView: input.GlossaryView,
Interactions: input.Interactions,
@@ -559,7 +553,7 @@ func runModulePipeline(ctx context.Context, input collectSectionProposalsInput)
if provider, ok := input.Module.(interface{ RepairPromptID() string }); ok {
promptID = provider.RepairPromptID()
}
validated, repairs, err = repairCandidates(runCtx, validationInput, sectionResult.references, validated, promptID)
validated, repairs, err = repairCandidates(runCtx, validationInput, sectionResult.proposal.SegmentReferences, validated, promptID)
}
setErr(err)
validationResults <- sectionValidationResult{

View File

@@ -20,7 +20,9 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
"gitea.maximumdirect.net/eric/audita/v3/internal/modules/grammar"
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
)
@@ -449,86 +451,6 @@ func TestRunnerProposalsExecutePerChunkSection(t *testing.T) {
}
}
func TestRunnerProposalIndexOrderingIsDeterministicAcrossParallelSections(t *testing.T) {
cfg := runnerTestConfig()
cfg.Chunking.MaxSectionTokens = 3
cfg.Chunking.MinSectionTokens = 0
transcript := &schema.Transcript{Segments: []schema.Segment{
{ID: 1, Text: "alpha one"},
{ID: 2, Text: "bravo two"},
{ID: 3, Text: "charlie three"},
}}
started := make(chan int, len(transcript.Segments))
releaseByID := map[int]chan struct{}{
1: make(chan struct{}),
2: make(chan struct{}),
3: make(chan struct{}),
}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
if len(req.WorkingTranscript.Segments) != 1 {
t.Fatalf("expected one segment per section, got %d", len(req.WorkingTranscript.Segments))
}
seg := req.WorkingTranscript.Segments[0]
started <- seg.ID
<-releaseByID[seg.ID]
return []proposals.CorrectionProposal{
{TargetSegmentID: seg.ID, OriginalText: seg.Text, CorrectedText: strings.ToUpper(seg.Text)},
}, nil
}},
}})
resultCh := make(chan struct {
out RunOutput
err error
}, 1)
go func() {
out, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: transcript,
ModuleSpecs: []contracts.ModuleRunSpec{
{ModuleKey: "m", InstanceName: "m"},
},
})
resultCh <- struct {
out RunOutput
err error
}{out: out, err: err}
}()
waitForRunnerSectionIDs(t, started, map[int]struct{}{1: {}, 2: {}, 3: {}})
close(releaseByID[3])
close(releaseByID[2])
close(releaseByID[1])
result := <-resultCh
out, err := result.out, result.err
if err != nil {
t.Fatalf("Run error: %v", err)
}
if len(out.ModuleResults) != 1 {
t.Fatalf("expected one module result, got %+v", out.ModuleResults)
}
if got := len(out.ModuleResults[0].AppliedChanges); got != 3 {
t.Fatalf("expected three applied changes, got %d", got)
}
for i, change := range out.ModuleResults[0].AppliedChanges {
wantID := i + 1
if change.TargetSegmentID != wantID {
t.Fatalf("expected deterministic applied-change order by section/segment, got %+v", out.ModuleResults[0].AppliedChanges)
}
}
wantFinal := []string{"ALPHA ONE", "BRAVO TWO", "CHARLIE THREE"}
for i, seg := range out.FinalTranscript.Segments {
if seg.Text != wantFinal[i] {
t.Fatalf("expected deterministic final transcript regardless of section completion order; got %+v", out.FinalTranscript.Segments)
}
}
}
func waitForRunnerEntries(t *testing.T, entered <-chan struct{}, want int, label string) {
t.Helper()
deadline := time.After(350 * time.Millisecond)
@@ -1228,90 +1150,117 @@ func TestRunnerLLMValidatorDiagnosticsWritten(t *testing.T) {
}
}
type proposalGenerationModule struct {
key string
policy proposals.ReplacementPolicy
type proposalClientFunc func(context.Context, contracts.StructuredCompletionRequest, any) (contracts.StructuredCompletionResponse, error)
func (f proposalClientFunc) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
return f(ctx, req, out)
}
func (m proposalGenerationModule) Key() string { return m.key }
func (m proposalGenerationModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy }
func (m proposalGenerationModule) Validators() []validators.Validator { return nil }
func (m proposalGenerationModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
result, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
ModuleKey: req.RunSpec.ModuleKey,
ModuleInstance: req.RunSpec.InstanceName,
ReplacementPolicy: req.RunSpec.ReplacementPolicy,
WorkingTranscript: req.WorkingTranscript,
PromptID: "modules.grammar.proposal",
Variables: map[string]string{"TranscriptDescriptionBlock": "", "GlossaryJSON": "[]", "SectionJSON": "[]"},
LLMClient: req.LLMClient,
Interactions: req.Interactions,
})
if err != nil {
return contracts.ProposalResult{}, err
}
return contracts.ProposalResult{Proposals: result.Corrections, Operations: result.Operations}, nil
}
type fakeProposalStructuredClient struct {
responses []proposal_generation.StructuredCorrectionSet
}
func (f *fakeProposalStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
_ = ctx
_ = req
if len(f.responses) == 0 {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected call")
}
target, ok := out.(*proposal_generation.StructuredCorrectionSet)
if !ok {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output type")
}
*target = f.responses[0]
f.responses = f.responses[1:]
return contracts.StructuredCompletionResponse{}, nil
}
func TestRunnerProposalGenerationHelperFlowsThroughPipeline(t *testing.T) {
client := &fakeProposalStructuredClient{
responses: []proposal_generation.StructuredCorrectionSet{
{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the"},
},
},
},
}
func TestRunnerProposalIndexOrderingIsDeterministicAcrossParallelSections(t *testing.T) {
cfg := runnerTestConfig()
diagDir := t.TempDir()
cfg.Chunking.MaxSectionTokens = 3
cfg.Chunking.MinSectionTokens = 0
transcript := &schema.Transcript{Segments: []schema.Segment{
{ID: 71, Text: "alpha one"},
{ID: 903, Text: "bravo two"},
}}
responses := []proposal_generation.StructuredCorrectionSet{
{Corrections: []proposal_generation.StructuredCorrectionProposal{
{SegmentReference: 1, OriginalText: "one", CorrectedText: "ONE"},
{SegmentReference: 1, OriginalText: "alpha", CorrectedText: "ALPHA"},
}},
{Corrections: []proposal_generation.StructuredCorrectionProposal{
{SegmentReference: 1, OriginalText: "two", CorrectedText: "TWO"},
}},
}
completedLaterSection := make(chan struct{})
client := proposalClientFunc(func(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
if req.Context.SectionIndex == nil {
return contracts.StructuredCompletionResponse{}, errors.New("missing section index")
}
index := *req.Context.SectionIndex
if index < 0 || index >= len(responses) {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected section index")
}
if index == 0 {
select {
case <-completedLaterSection:
case <-ctx.Done():
return contracts.StructuredCompletionResponse{}, ctx.Err()
}
}
*out.(*proposal_generation.StructuredCorrectionSet) = responses[index]
return contracts.StructuredCompletionResponse{}, nil
})
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": proposalGenerationModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique},
}})
out, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ProposalClient: client, Settings: config.Stage{}}},
Interactions: llm.NewDiagnosticsWriter(diagDir),
dir := t.TempDir()
writer := llm.NewDiagnosticsWriter(dir)
interactions := proposalOrderSink{writer: writer, laterResponse: completedLaterSection}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": &grammar.Module{}}})
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
out, err := r.Run(ctx, RunInput{
Config: &cfg, Transcript: transcript, Interactions: interactions,
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", ProposalClient: client}},
})
if err != nil {
t.Fatalf("unexpected run error: %v", err)
t.Fatal(err)
}
if out.FinalTranscript.Segments[0].Text != "the cat" {
t.Fatalf("expected proposal-generated correction to apply, got %q", out.FinalTranscript.Segments[0].Text)
want := []struct {
target int
original string
}{{71, "one"}, {71, "alpha"}, {903, "two"}}
changes := out.ModuleResults[0].AppliedChanges
if len(changes) != len(want) {
t.Fatalf("applied changes: %+v", changes)
}
if len(out.ModuleResults) != 1 || len(out.ModuleResults[0].AppliedChanges) != 1 {
t.Fatalf("expected one applied change, got %+v", out.ModuleResults)
for i, change := range changes {
if change.ProposalIndex != i || change.TargetSegmentID != want[i].target || change.OriginalText != want[i].original {
t.Fatalf("change %d lost response order or segment identity: %+v", i, change)
}
}
matches, globErr := filepath.Glob(filepath.Join(diagDir, "m", "*proposal-generation*response-payload.json"))
if globErr != nil {
t.Fatalf("glob diagnostics: %v", globErr)
wantIndices := [][]int{{0, 1}, {2}}
if len(out.WorkUnits) != len(wantIndices) {
t.Fatalf("work units: %+v", out.WorkUnits)
}
if len(matches) == 0 {
t.Fatalf("expected proposal-generation diagnostics artifacts in %s", filepath.Join(diagDir, "m"))
for i, unit := range out.WorkUnits {
if len(unit.Operations) != 1 || !reflect.DeepEqual(unit.Operations[0].ProposalIndices, wantIndices[i]) {
t.Fatalf("section %d lost proposal indices: %+v", i, unit.Operations)
}
}
paths, err := filepath.Glob(filepath.Join(dir, "grammar", "*response-payload.json"))
if err != nil || len(paths) != len(responses) {
t.Fatalf("proposal diagnostic paths=%v error=%v", paths, err)
}
for i, path := range paths {
raw, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
var evidence struct {
Decoded proposal_generation.StructuredCorrectionSet `json:"decoded"`
}
if err := json.Unmarshal(raw, &evidence); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(evidence.Decoded, responses[i]) {
t.Fatalf("section %d lost original references: %+v", i, evidence.Decoded)
}
}
}
type proposalOrderSink struct {
writer *llm.DiagnosticsWriter
laterResponse chan struct{}
}
func (s proposalOrderSink) WriteInteraction(interaction structuredoutput.Interaction) (structuredoutput.InteractionArtifacts, error) {
artifacts, err := s.writer.WriteInteraction(interaction)
if interaction.Context.SectionIndex != nil && *interaction.Context.SectionIndex == 1 {
close(s.laterResponse)
}
return artifacts, err
}
func TestFatalPartialDecisionsRejectUnknownAndDuplicateIdentity(t *testing.T) {