Carry one proposal result from generation to the runner

This commit is contained in:
2026-11-16 14:30:57 +00:00
parent c2c8050bc8
commit a0c70e8dce
9 changed files with 153 additions and 314 deletions

View File

@@ -73,10 +73,10 @@ func TestRepairedProposalStillFacesDomainChecks(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if calls != 2 || len(generated.Enriched) != 1 {
if calls != 2 || len(generated.Proposals) != 1 {
t.Fatalf("repair path: calls=%d, result=%+v", calls, generated)
}
input := validators.Request{ModuleKey: "grammar", Settings: config.Stage{}, CandidateProposal: generated.Enriched, WorkingTranscript: transcript}
input := validators.Request{ModuleKey: "grammar", Settings: config.Stage{}, CandidateProposal: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: generated.Proposals[0]}}, WorkingTranscript: transcript}
missing, err := (validators.OriginalTextPresenceValidator{}).Validate(context.Background(), input)
if err != nil {
t.Fatal(err)

View File

@@ -42,7 +42,6 @@ type Request struct {
PromptID string `json:"prompt_id"`
Variables map[string]string `json:"variables"`
StageName string `json:"stage_name,omitempty"`
StartIndex int `json:"start_index"`
CaptureExchange bool
LLMClient contracts.StructuredLLMClient
Interactions structuredoutput.InteractionSink
@@ -50,12 +49,8 @@ type Request struct {
// Result contains generated candidate proposals and optional diagnostics paths.
type Result struct {
SegmentReferences []int `json:"-"`
Exchange *structuredoutput.Exchange `json:"-"`
Corrections []proposals.CorrectionProposal `json:"corrections"`
Enriched []proposals.EnrichedCorrectionProposal `json:"enriched"`
Operations []outcomes.Operation `json:"operations,omitempty"`
Artifacts structuredoutput.InteractionArtifacts `json:"artifacts,omitempty"`
contracts.ProposalResult
Artifacts structuredoutput.InteractionArtifacts `json:"artifacts,omitempty"`
}
// GenerateCandidates executes one structured LLM call and deterministically maps
@@ -67,9 +62,6 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
if strings.TrimSpace(req.ModuleInstance) == "" {
return Result{}, fmt.Errorf("module instance must not be empty")
}
if req.StartIndex < 0 {
return Result{}, fmt.Errorf("start index must be non-negative")
}
if req.LLMClient == nil {
return Result{}, fmt.Errorf("structured LLM client is required")
}
@@ -115,7 +107,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
"module_instance": req.ModuleInstance,
"replacement_policy": req.ReplacementPolicy,
"section": req.Section,
"start_index": req.StartIndex,
"start_index": 0, // Position of the first item within this response.
"segment_ids": segmentIDs(req.WorkingTranscript),
"prompt": completion.Prompt,
}
@@ -129,7 +121,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
if callErr != nil {
structuredoutput.RecordFailure(&operation, callErr)
result := Result{Operations: []outcomes.Operation{operation}, Artifacts: artifacts}
result := Result{ProposalResult: contracts.ProposalResult{Operations: []outcomes.Operation{operation}}, Artifacts: artifacts}
if operation.FailureClass == outcomes.FailureRecoverable {
return result, nil
}
@@ -140,39 +132,23 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
references := make([]int, 0, len(response.Corrections))
corrections := make([]proposals.CorrectionProposal, 0, len(response.Corrections))
enriched := make([]proposals.EnrichedCorrectionProposal, 0, len(response.Corrections))
for i, raw := range response.Corrections {
for _, raw := range response.Corrections {
references = append(references, raw.SegmentReference)
// Unknown references remain invalid candidates for the existing domain validators.
candidate := proposals.CorrectionProposal{
corrections = append(corrections, proposals.CorrectionProposal{
TargetSegmentID: proposals.SegmentIDAtReference(req.WorkingTranscript, raw.SegmentReference),
OriginalText: raw.OriginalText,
CorrectedText: raw.CorrectedText,
}
corrections = append(corrections, candidate)
enrichedCandidate := proposals.EnrichedCorrectionProposal{
CorrectionProposal: candidate,
ProposalMetadata: proposals.ProposalMetadata{
ProposalIndex: req.StartIndex + i,
ModuleKey: req.ModuleKey,
ModuleInstance: req.ModuleInstance,
},
}
if req.Section != nil {
sectionIndex := req.Section.Index
enrichedCandidate.SectionIndex = &sectionIndex
}
enriched = append(enriched, enrichedCandidate)
})
}
return Result{
Corrections: corrections,
SegmentReferences: references,
Exchange: completion.Exchange,
Enriched: enriched,
Operations: []outcomes.Operation{operation},
Artifacts: artifacts,
ProposalResult: contracts.ProposalResult{
Proposals: corrections,
SegmentReferences: references,
Exchange: completion.Exchange,
Operations: []outcomes.Operation{operation},
},
Artifacts: artifacts,
}, nil
}

View File

@@ -6,7 +6,6 @@ import (
"errors"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
@@ -73,7 +72,6 @@ func defaultRequest(t *testing.T) Request {
PromptID: "modules.grammar.proposal",
Variables: map[string]string{"SectionJSON": "[]"},
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 7, Text: "teh"}, {ID: 25, Text: "other"}}},
StartIndex: 0,
}
}
@@ -90,24 +88,17 @@ func TestGenerateCandidatesSuccess(t *testing.T) {
section := contracts.SectionMetadata{Index: 2}
req := defaultRequest(t)
req.LLMClient = client
req.StartIndex = 10
req.Section = &section
got, err := GenerateCandidates(context.Background(), req)
if err != nil {
t.Fatalf("GenerateCandidates error: %v", err)
}
if len(got.Corrections) != 1 || len(got.Enriched) != 1 {
if len(got.Proposals) != 1 || len(got.SegmentReferences) != 1 || got.SegmentReferences[0] != 1 {
t.Fatalf("unexpected proposal lengths: %+v", got)
}
if got.Corrections[0].TargetSegmentID != 7 {
t.Fatalf("local reference not resolved: %+v", got.Corrections)
}
if got.Enriched[0].ProposalIndex != 10 {
t.Fatalf("expected proposal index 10, got %d", got.Enriched[0].ProposalIndex)
}
if got.Enriched[0].SectionIndex == nil || *got.Enriched[0].SectionIndex != 2 {
t.Fatalf("expected section index 2, got %v", got.Enriched[0].SectionIndex)
if got.Proposals[0].TargetSegmentID != 7 {
t.Fatalf("local reference not resolved: %+v", got.Proposals)
}
if len(got.Operations) != 1 || !got.Operations[0].Succeeded || got.Operations[0].Reference.SectionIndex != 2 || got.Operations[0].Reference.Kind != outcomes.OperationProposal {
t.Fatalf("proposal operation identity: %+v", got.Operations)
@@ -184,11 +175,11 @@ func TestGenerateCandidatesInvalidCorrectionIsPreservedForLaterValidation(t *tes
if err != nil {
t.Fatalf("expected invalid correction to survive generation, got %v", err)
}
if len(result.Corrections) != 1 {
if len(result.Proposals) != 1 {
t.Fatalf("expected one correction, got %+v", result)
}
if result.Corrections[0].TargetSegmentID != 0 {
t.Fatalf("unexpected preserved correction: %+v", result.Corrections[0])
if result.Proposals[0].TargetSegmentID != 0 {
t.Fatalf("unexpected preserved correction: %+v", result.Proposals[0])
}
raw, err := os.ReadFile(result.Artifacts.ResponsePayloadPath)
if err != nil {
@@ -218,7 +209,7 @@ func TestGenerateCandidatesMalformedStructuredOutputReturnsFailedOperation(t *te
if err != nil {
t.Fatalf("expected recoverable model operation, got %v", err)
}
if len(result.Corrections) != 0 || len(result.Enriched) != 0 {
if len(result.Proposals) != 0 {
t.Fatalf("expected no proposals on malformed response, got %+v", result)
}
if len(result.Operations) != 1 || result.Operations[0].Succeeded || result.Operations[0].FailureReason != outcomes.ReasonInvalidOutputExhausted {
@@ -255,87 +246,11 @@ func TestGenerateCandidatesValidEmptyResponseIsSuccessful(t *testing.T) {
req := defaultRequest(t)
req.LLMClient = client
result, err := GenerateCandidates(context.Background(), req)
if err != nil || len(result.Corrections) != 0 || len(result.Operations) != 1 || !result.Operations[0].Succeeded {
if err != nil || len(result.Proposals) != 0 || len(result.Operations) != 1 || !result.Operations[0].Succeeded {
t.Fatalf("valid empty response: %+v, %v", result, err)
}
}
func TestGenerateCandidatesDeterministicIndexAssignment(t *testing.T) {
baseResponse := StructuredCorrectionSet{
Corrections: []StructuredCorrectionProposal{
{SegmentReference: 1, OriginalText: "a", CorrectedText: "A"},
{SegmentReference: 2, OriginalText: "b", CorrectedText: "B"},
},
}
clientA := &fakeStructuredClient{responses: []StructuredCorrectionSet{baseResponse}}
clientB := &fakeStructuredClient{responses: []StructuredCorrectionSet{baseResponse}}
reqA := defaultRequest(t)
reqA.LLMClient = clientA
reqA.StartIndex = 3
first, err := GenerateCandidates(context.Background(), reqA)
if err != nil {
t.Fatalf("first generation failed: %v", err)
}
reqB := defaultRequest(t)
reqB.LLMClient = clientB
reqB.StartIndex = 3
second, err := GenerateCandidates(context.Background(), reqB)
if err != nil {
t.Fatalf("second generation failed: %v", err)
}
if !reflect.DeepEqual(first.Enriched, second.Enriched) {
t.Fatalf("expected stable enriched proposals\nfirst=%+v\nsecond=%+v", first.Enriched, second.Enriched)
}
}
func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) {
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "alpha", CorrectedText: "ALPHA"}}},
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "charlie", CorrectedText: "CHARLIE"}}},
},
}
section0 := contracts.SectionMetadata{Index: 0}
section1 := contracts.SectionMetadata{Index: 1}
req0 := defaultRequest(t)
req0.LLMClient = client
req0.Section = &section0
req0.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "alpha"}}}
req0.StartIndex = 0
part0, err := GenerateCandidates(context.Background(), req0)
if err != nil {
t.Fatalf("section 0 generation failed: %v", err)
}
req1 := defaultRequest(t)
req1.LLMClient = client
req1.Section = &section1
req1.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{{ID: 903, Text: "charlie"}}}
req1.StartIndex = len(part0.Enriched)
part1, err := GenerateCandidates(context.Background(), req1)
if err != nil {
t.Fatalf("section 1 generation failed: %v", err)
}
all := append(append([]proposals.EnrichedCorrectionProposal(nil), part0.Enriched...), part1.Enriched...)
if len(all) != 2 {
t.Fatalf("expected 2 proposals, got %d", len(all))
}
if all[0].TargetSegmentID != 71 || all[1].TargetSegmentID != 903 {
t.Fatalf("section references did not reset: %+v", all)
}
if all[0].ProposalIndex != 0 || all[1].ProposalIndex != 1 {
t.Fatalf("unexpected proposal indexes: %d, %d", all[0].ProposalIndex, all[1].ProposalIndex)
}
if all[0].SectionIndex == nil || *all[0].SectionIndex != 0 || all[1].SectionIndex == nil || *all[1].SectionIndex != 1 {
t.Fatalf("unexpected section metadata: %+v", all)
}
}
func TestGenerateCandidatesDiagnosticsWritten(t *testing.T) {
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
@@ -375,6 +290,9 @@ func TestGenerateCandidatesDiagnosticsWritten(t *testing.T) {
if err := json.Unmarshal(raw, &metadata); err != nil {
t.Fatalf("unmarshal metadata artifact: %v", err)
}
if start, ok := metadata["start_index"].(float64); !ok || start != 0 {
t.Fatalf("missing response-local origin: %s", raw)
}
schemaMap, ok := metadata["response_schema"].(map[string]any)
if !ok {
t.Fatalf("expected response_schema metadata in diagnostics, got %T", metadata["response_schema"])
@@ -405,7 +323,7 @@ func TestGenerateCandidatesClientError(t *testing.T) {
if err == nil || !strings.Contains(err.Error(), "completion failed") {
t.Fatalf("expected completion failure, got %v", err)
}
if len(got.Operations) != 1 || got.Operations[0].FailureReason != outcomes.ReasonInternalFailure || len(got.Corrections) != 0 {
if len(got.Operations) != 1 || got.Operations[0].FailureReason != outcomes.ReasonInternalFailure || len(got.Proposals) != 0 {
t.Fatalf("expected fatal call facts without proposals, got %+v", got)
}
matches, globErr := filepath.Glob(filepath.Join(dir, req.ModuleInstance, "*error-payload.json"))

View File

@@ -62,19 +62,9 @@ func ExecuteModuleProposal(ctx context.Context, req ModuleProposalRequest) (cont
PromptID: promptMetadata.PromptID,
Variables: variables,
StageName: stagename.ModuleProposal(req.ProposalRequest.RunSpec.InstanceName, sectionIndexPtr(req.ProposalRequest.Section)),
StartIndex: 0,
CaptureExchange: req.ProposalRequest.RunSpec.CaptureProposalExchange,
LLMClient: req.ProposalRequest.LLMClient,
Interactions: req.ProposalRequest.Interactions,
})
if err != nil {
return contracts.ProposalResult{Operations: generated.Operations}, err
}
return contracts.ProposalResult{
Proposals: generated.Corrections,
SegmentReferences: generated.SegmentReferences,
Exchange: generated.Exchange,
Operations: generated.Operations,
}, nil
return generated.ProposalResult, err
}