Carry one proposal result from generation to the runner
This commit is contained in:
@@ -73,10 +73,10 @@ func TestRepairedProposalStillFacesDomainChecks(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if calls != 2 || len(generated.Enriched) != 1 {
|
||||
if calls != 2 || len(generated.Proposals) != 1 {
|
||||
t.Fatalf("repair path: calls=%d, result=%+v", calls, generated)
|
||||
}
|
||||
input := validators.Request{ModuleKey: "grammar", Settings: config.Stage{}, CandidateProposal: generated.Enriched, WorkingTranscript: transcript}
|
||||
input := validators.Request{ModuleKey: "grammar", Settings: config.Stage{}, CandidateProposal: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: generated.Proposals[0]}}, WorkingTranscript: transcript}
|
||||
missing, err := (validators.OriginalTextPresenceValidator{}).Validate(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -42,7 +42,6 @@ type Request struct {
|
||||
PromptID string `json:"prompt_id"`
|
||||
Variables map[string]string `json:"variables"`
|
||||
StageName string `json:"stage_name,omitempty"`
|
||||
StartIndex int `json:"start_index"`
|
||||
CaptureExchange bool
|
||||
LLMClient contracts.StructuredLLMClient
|
||||
Interactions structuredoutput.InteractionSink
|
||||
@@ -50,12 +49,8 @@ type Request struct {
|
||||
|
||||
// Result contains generated candidate proposals and optional diagnostics paths.
|
||||
type Result struct {
|
||||
SegmentReferences []int `json:"-"`
|
||||
Exchange *structuredoutput.Exchange `json:"-"`
|
||||
Corrections []proposals.CorrectionProposal `json:"corrections"`
|
||||
Enriched []proposals.EnrichedCorrectionProposal `json:"enriched"`
|
||||
Operations []outcomes.Operation `json:"operations,omitempty"`
|
||||
Artifacts structuredoutput.InteractionArtifacts `json:"artifacts,omitempty"`
|
||||
contracts.ProposalResult
|
||||
Artifacts structuredoutput.InteractionArtifacts `json:"artifacts,omitempty"`
|
||||
}
|
||||
|
||||
// GenerateCandidates executes one structured LLM call and deterministically maps
|
||||
@@ -67,9 +62,6 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
if strings.TrimSpace(req.ModuleInstance) == "" {
|
||||
return Result{}, fmt.Errorf("module instance must not be empty")
|
||||
}
|
||||
if req.StartIndex < 0 {
|
||||
return Result{}, fmt.Errorf("start index must be non-negative")
|
||||
}
|
||||
if req.LLMClient == nil {
|
||||
return Result{}, fmt.Errorf("structured LLM client is required")
|
||||
}
|
||||
@@ -115,7 +107,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
"module_instance": req.ModuleInstance,
|
||||
"replacement_policy": req.ReplacementPolicy,
|
||||
"section": req.Section,
|
||||
"start_index": req.StartIndex,
|
||||
"start_index": 0, // Position of the first item within this response.
|
||||
"segment_ids": segmentIDs(req.WorkingTranscript),
|
||||
"prompt": completion.Prompt,
|
||||
}
|
||||
@@ -129,7 +121,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
|
||||
if callErr != nil {
|
||||
structuredoutput.RecordFailure(&operation, callErr)
|
||||
result := Result{Operations: []outcomes.Operation{operation}, Artifacts: artifacts}
|
||||
result := Result{ProposalResult: contracts.ProposalResult{Operations: []outcomes.Operation{operation}}, Artifacts: artifacts}
|
||||
if operation.FailureClass == outcomes.FailureRecoverable {
|
||||
return result, nil
|
||||
}
|
||||
@@ -140,39 +132,23 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
|
||||
references := make([]int, 0, len(response.Corrections))
|
||||
corrections := make([]proposals.CorrectionProposal, 0, len(response.Corrections))
|
||||
enriched := make([]proposals.EnrichedCorrectionProposal, 0, len(response.Corrections))
|
||||
for i, raw := range response.Corrections {
|
||||
for _, raw := range response.Corrections {
|
||||
references = append(references, raw.SegmentReference)
|
||||
// Unknown references remain invalid candidates for the existing domain validators.
|
||||
candidate := proposals.CorrectionProposal{
|
||||
corrections = append(corrections, proposals.CorrectionProposal{
|
||||
TargetSegmentID: proposals.SegmentIDAtReference(req.WorkingTranscript, raw.SegmentReference),
|
||||
OriginalText: raw.OriginalText,
|
||||
CorrectedText: raw.CorrectedText,
|
||||
}
|
||||
|
||||
corrections = append(corrections, candidate)
|
||||
enrichedCandidate := proposals.EnrichedCorrectionProposal{
|
||||
CorrectionProposal: candidate,
|
||||
ProposalMetadata: proposals.ProposalMetadata{
|
||||
ProposalIndex: req.StartIndex + i,
|
||||
ModuleKey: req.ModuleKey,
|
||||
ModuleInstance: req.ModuleInstance,
|
||||
},
|
||||
}
|
||||
if req.Section != nil {
|
||||
sectionIndex := req.Section.Index
|
||||
enrichedCandidate.SectionIndex = §ionIndex
|
||||
}
|
||||
enriched = append(enriched, enrichedCandidate)
|
||||
})
|
||||
}
|
||||
|
||||
return Result{
|
||||
Corrections: corrections,
|
||||
SegmentReferences: references,
|
||||
Exchange: completion.Exchange,
|
||||
Enriched: enriched,
|
||||
Operations: []outcomes.Operation{operation},
|
||||
Artifacts: artifacts,
|
||||
ProposalResult: contracts.ProposalResult{
|
||||
Proposals: corrections,
|
||||
SegmentReferences: references,
|
||||
Exchange: completion.Exchange,
|
||||
Operations: []outcomes.Operation{operation},
|
||||
},
|
||||
Artifacts: artifacts,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -73,7 +72,6 @@ func defaultRequest(t *testing.T) Request {
|
||||
PromptID: "modules.grammar.proposal",
|
||||
Variables: map[string]string{"SectionJSON": "[]"},
|
||||
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 7, Text: "teh"}, {ID: 25, Text: "other"}}},
|
||||
StartIndex: 0,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -90,24 +88,17 @@ func TestGenerateCandidatesSuccess(t *testing.T) {
|
||||
section := contracts.SectionMetadata{Index: 2}
|
||||
req := defaultRequest(t)
|
||||
req.LLMClient = client
|
||||
req.StartIndex = 10
|
||||
req.Section = §ion
|
||||
|
||||
got, err := GenerateCandidates(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("GenerateCandidates error: %v", err)
|
||||
}
|
||||
if len(got.Corrections) != 1 || len(got.Enriched) != 1 {
|
||||
if len(got.Proposals) != 1 || len(got.SegmentReferences) != 1 || got.SegmentReferences[0] != 1 {
|
||||
t.Fatalf("unexpected proposal lengths: %+v", got)
|
||||
}
|
||||
if got.Corrections[0].TargetSegmentID != 7 {
|
||||
t.Fatalf("local reference not resolved: %+v", got.Corrections)
|
||||
}
|
||||
if got.Enriched[0].ProposalIndex != 10 {
|
||||
t.Fatalf("expected proposal index 10, got %d", got.Enriched[0].ProposalIndex)
|
||||
}
|
||||
if got.Enriched[0].SectionIndex == nil || *got.Enriched[0].SectionIndex != 2 {
|
||||
t.Fatalf("expected section index 2, got %v", got.Enriched[0].SectionIndex)
|
||||
if got.Proposals[0].TargetSegmentID != 7 {
|
||||
t.Fatalf("local reference not resolved: %+v", got.Proposals)
|
||||
}
|
||||
if len(got.Operations) != 1 || !got.Operations[0].Succeeded || got.Operations[0].Reference.SectionIndex != 2 || got.Operations[0].Reference.Kind != outcomes.OperationProposal {
|
||||
t.Fatalf("proposal operation identity: %+v", got.Operations)
|
||||
@@ -184,11 +175,11 @@ func TestGenerateCandidatesInvalidCorrectionIsPreservedForLaterValidation(t *tes
|
||||
if err != nil {
|
||||
t.Fatalf("expected invalid correction to survive generation, got %v", err)
|
||||
}
|
||||
if len(result.Corrections) != 1 {
|
||||
if len(result.Proposals) != 1 {
|
||||
t.Fatalf("expected one correction, got %+v", result)
|
||||
}
|
||||
if result.Corrections[0].TargetSegmentID != 0 {
|
||||
t.Fatalf("unexpected preserved correction: %+v", result.Corrections[0])
|
||||
if result.Proposals[0].TargetSegmentID != 0 {
|
||||
t.Fatalf("unexpected preserved correction: %+v", result.Proposals[0])
|
||||
}
|
||||
raw, err := os.ReadFile(result.Artifacts.ResponsePayloadPath)
|
||||
if err != nil {
|
||||
@@ -218,7 +209,7 @@ func TestGenerateCandidatesMalformedStructuredOutputReturnsFailedOperation(t *te
|
||||
if err != nil {
|
||||
t.Fatalf("expected recoverable model operation, got %v", err)
|
||||
}
|
||||
if len(result.Corrections) != 0 || len(result.Enriched) != 0 {
|
||||
if len(result.Proposals) != 0 {
|
||||
t.Fatalf("expected no proposals on malformed response, got %+v", result)
|
||||
}
|
||||
if len(result.Operations) != 1 || result.Operations[0].Succeeded || result.Operations[0].FailureReason != outcomes.ReasonInvalidOutputExhausted {
|
||||
@@ -255,87 +246,11 @@ func TestGenerateCandidatesValidEmptyResponseIsSuccessful(t *testing.T) {
|
||||
req := defaultRequest(t)
|
||||
req.LLMClient = client
|
||||
result, err := GenerateCandidates(context.Background(), req)
|
||||
if err != nil || len(result.Corrections) != 0 || len(result.Operations) != 1 || !result.Operations[0].Succeeded {
|
||||
if err != nil || len(result.Proposals) != 0 || len(result.Operations) != 1 || !result.Operations[0].Succeeded {
|
||||
t.Fatalf("valid empty response: %+v, %v", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateCandidatesDeterministicIndexAssignment(t *testing.T) {
|
||||
baseResponse := StructuredCorrectionSet{
|
||||
Corrections: []StructuredCorrectionProposal{
|
||||
{SegmentReference: 1, OriginalText: "a", CorrectedText: "A"},
|
||||
{SegmentReference: 2, OriginalText: "b", CorrectedText: "B"},
|
||||
},
|
||||
}
|
||||
clientA := &fakeStructuredClient{responses: []StructuredCorrectionSet{baseResponse}}
|
||||
clientB := &fakeStructuredClient{responses: []StructuredCorrectionSet{baseResponse}}
|
||||
|
||||
reqA := defaultRequest(t)
|
||||
reqA.LLMClient = clientA
|
||||
reqA.StartIndex = 3
|
||||
first, err := GenerateCandidates(context.Background(), reqA)
|
||||
if err != nil {
|
||||
t.Fatalf("first generation failed: %v", err)
|
||||
}
|
||||
|
||||
reqB := defaultRequest(t)
|
||||
reqB.LLMClient = clientB
|
||||
reqB.StartIndex = 3
|
||||
second, err := GenerateCandidates(context.Background(), reqB)
|
||||
if err != nil {
|
||||
t.Fatalf("second generation failed: %v", err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(first.Enriched, second.Enriched) {
|
||||
t.Fatalf("expected stable enriched proposals\nfirst=%+v\nsecond=%+v", first.Enriched, second.Enriched)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) {
|
||||
client := &fakeStructuredClient{
|
||||
responses: []StructuredCorrectionSet{
|
||||
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "alpha", CorrectedText: "ALPHA"}}},
|
||||
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "charlie", CorrectedText: "CHARLIE"}}},
|
||||
},
|
||||
}
|
||||
section0 := contracts.SectionMetadata{Index: 0}
|
||||
section1 := contracts.SectionMetadata{Index: 1}
|
||||
|
||||
req0 := defaultRequest(t)
|
||||
req0.LLMClient = client
|
||||
req0.Section = §ion0
|
||||
req0.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "alpha"}}}
|
||||
req0.StartIndex = 0
|
||||
part0, err := GenerateCandidates(context.Background(), req0)
|
||||
if err != nil {
|
||||
t.Fatalf("section 0 generation failed: %v", err)
|
||||
}
|
||||
|
||||
req1 := defaultRequest(t)
|
||||
req1.LLMClient = client
|
||||
req1.Section = §ion1
|
||||
req1.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{{ID: 903, Text: "charlie"}}}
|
||||
req1.StartIndex = len(part0.Enriched)
|
||||
part1, err := GenerateCandidates(context.Background(), req1)
|
||||
if err != nil {
|
||||
t.Fatalf("section 1 generation failed: %v", err)
|
||||
}
|
||||
|
||||
all := append(append([]proposals.EnrichedCorrectionProposal(nil), part0.Enriched...), part1.Enriched...)
|
||||
if len(all) != 2 {
|
||||
t.Fatalf("expected 2 proposals, got %d", len(all))
|
||||
}
|
||||
if all[0].TargetSegmentID != 71 || all[1].TargetSegmentID != 903 {
|
||||
t.Fatalf("section references did not reset: %+v", all)
|
||||
}
|
||||
if all[0].ProposalIndex != 0 || all[1].ProposalIndex != 1 {
|
||||
t.Fatalf("unexpected proposal indexes: %d, %d", all[0].ProposalIndex, all[1].ProposalIndex)
|
||||
}
|
||||
if all[0].SectionIndex == nil || *all[0].SectionIndex != 0 || all[1].SectionIndex == nil || *all[1].SectionIndex != 1 {
|
||||
t.Fatalf("unexpected section metadata: %+v", all)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateCandidatesDiagnosticsWritten(t *testing.T) {
|
||||
client := &fakeStructuredClient{
|
||||
responses: []StructuredCorrectionSet{
|
||||
@@ -375,6 +290,9 @@ func TestGenerateCandidatesDiagnosticsWritten(t *testing.T) {
|
||||
if err := json.Unmarshal(raw, &metadata); err != nil {
|
||||
t.Fatalf("unmarshal metadata artifact: %v", err)
|
||||
}
|
||||
if start, ok := metadata["start_index"].(float64); !ok || start != 0 {
|
||||
t.Fatalf("missing response-local origin: %s", raw)
|
||||
}
|
||||
schemaMap, ok := metadata["response_schema"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected response_schema metadata in diagnostics, got %T", metadata["response_schema"])
|
||||
@@ -405,7 +323,7 @@ func TestGenerateCandidatesClientError(t *testing.T) {
|
||||
if err == nil || !strings.Contains(err.Error(), "completion failed") {
|
||||
t.Fatalf("expected completion failure, got %v", err)
|
||||
}
|
||||
if len(got.Operations) != 1 || got.Operations[0].FailureReason != outcomes.ReasonInternalFailure || len(got.Corrections) != 0 {
|
||||
if len(got.Operations) != 1 || got.Operations[0].FailureReason != outcomes.ReasonInternalFailure || len(got.Proposals) != 0 {
|
||||
t.Fatalf("expected fatal call facts without proposals, got %+v", got)
|
||||
}
|
||||
matches, globErr := filepath.Glob(filepath.Join(dir, req.ModuleInstance, "*error-payload.json"))
|
||||
|
||||
@@ -62,19 +62,9 @@ func ExecuteModuleProposal(ctx context.Context, req ModuleProposalRequest) (cont
|
||||
PromptID: promptMetadata.PromptID,
|
||||
Variables: variables,
|
||||
StageName: stagename.ModuleProposal(req.ProposalRequest.RunSpec.InstanceName, sectionIndexPtr(req.ProposalRequest.Section)),
|
||||
StartIndex: 0,
|
||||
CaptureExchange: req.ProposalRequest.RunSpec.CaptureProposalExchange,
|
||||
LLMClient: req.ProposalRequest.LLMClient,
|
||||
Interactions: req.ProposalRequest.Interactions,
|
||||
})
|
||||
if err != nil {
|
||||
return contracts.ProposalResult{Operations: generated.Operations}, err
|
||||
}
|
||||
|
||||
return contracts.ProposalResult{
|
||||
Proposals: generated.Corrections,
|
||||
SegmentReferences: generated.SegmentReferences,
|
||||
Exchange: generated.Exchange,
|
||||
Operations: generated.Operations,
|
||||
}, nil
|
||||
return generated.ProposalResult, err
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user