Share repair histories while debug artifacts wait for indices
This commit is contained in:
@@ -10,6 +10,7 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
@@ -194,10 +195,11 @@ func TestLaterRepairArtifactsUseOrderedProposalIndices(t *testing.T) {
|
||||
cancel()
|
||||
return contracts.StructuredCompletionResponse{}, ctx.Err()
|
||||
}
|
||||
lastRevision := revision && *req.Context.RepairAttempt == 2
|
||||
dst.(*validators.LLMValidationResponse).Validations = []validators.LLMValidationDecision{{
|
||||
CorrectionIndex: "1", Approved: first || revision, Reason: "checked",
|
||||
CorrectionIndex: "1", Approved: first || lastRevision, Reason: "checked",
|
||||
}}
|
||||
if revision {
|
||||
if lastRevision {
|
||||
close(laterRevalidated)
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{}, nil
|
||||
@@ -211,18 +213,30 @@ func TestLaterRepairArtifactsUseOrderedProposalIndices(t *testing.T) {
|
||||
started := make(chan struct{})
|
||||
close(started)
|
||||
module.headEntered = started
|
||||
repair := &repairClient{t: t, outputs: []string{
|
||||
var captured []*structuredoutput.PreparedSnapshot
|
||||
repair := &repairClient{t: t, capturePrepared: func(p *structuredoutput.PreparedSnapshot) {
|
||||
captured = append(captured, p)
|
||||
}, outputs: []string{
|
||||
`{"proposal_ref":1,"decision":{"action":"revise","correction":{"id":1,"original_text":"bravo","corrected_text":"BRAVO"}}}`,
|
||||
`{"proposal_ref":1,"decision":{"action":"revise","correction":{"id":1,"original_text":"bravo","corrected_text":"Bravo"}}}`,
|
||||
}}
|
||||
dir := t.TempDir()
|
||||
if canceled {
|
||||
blocked := filepath.Join(dir, "grammar", "grammar_section-0001_proposal-0001_attempt-0001_repair-request-payload.json")
|
||||
if err := os.MkdirAll(blocked, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
writer := llm.NewDiagnosticsWriter(dir)
|
||||
cfg := runnerTestConfig()
|
||||
cfg.Chunking.MaxSectionTokens = 1
|
||||
cfg.Chunking.MinSectionTokens = 0
|
||||
out, err := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}}).Run(ctx, RunInput{
|
||||
Config: &cfg, Interactions: llm.NewDiagnosticsWriter(t.TempDir()), Evidence: &EvidenceSelection{Reports: true, Debug: true},
|
||||
Config: &cfg, Interactions: writer, Evidence: &EvidenceSelection{Reports: true, Debug: true},
|
||||
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "alpha"}, {ID: 93, Text: "bravo"}}},
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{{
|
||||
ModuleKey: "grammar", InstanceName: "grammar", ProposalClient: proposal, ValidationClient: validation,
|
||||
RepairCoordinator: repair, Settings: config.Stage{Validation: cfg.Validation, LLM: config.LLM{CorrectionRepairAttempts: 1}},
|
||||
RepairCoordinator: repair, Settings: config.Stage{Validation: cfg.Validation, LLM: config.LLM{CorrectionRepairAttempts: 2}},
|
||||
}},
|
||||
})
|
||||
if (!canceled && err != nil) || (canceled && !errors.Is(err, context.Canceled)) {
|
||||
@@ -230,10 +244,26 @@ func TestLaterRepairArtifactsUseOrderedProposalIndices(t *testing.T) {
|
||||
}
|
||||
later := out.WorkUnits[1]
|
||||
if len(out.Candidates) != 2 || out.Candidates[1].ProposalIndex != 1 ||
|
||||
len(later.Repairs) != 1 || later.Repairs[0].ProposalIndex != 1 || len(later.Repairs[0].Attempts) != 1 {
|
||||
len(later.Repairs) != 1 || later.Repairs[0].ProposalIndex != 1 || len(later.Repairs[0].Attempts) != 2 {
|
||||
t.Fatalf("lost ordered candidate or repair: %+v, %+v", out.Candidates, later.Repairs)
|
||||
}
|
||||
attempt := later.Repairs[0].Attempts[0]
|
||||
if len(captured) != 2 || len(captured[1].Messages) <= len(captured[0].Messages) {
|
||||
t.Fatalf("missing growing repair history: %+v", captured)
|
||||
}
|
||||
for i, attempt := range later.Repairs[0].Attempts {
|
||||
if i == 0 && canceled {
|
||||
if attempt.Artifacts.RequestPayloadPath != "" || len(writer.Warnings()) != 1 {
|
||||
t.Fatalf("expected one optional request write failure: %+v, %v", attempt.Artifacts, writer.Warnings())
|
||||
}
|
||||
continue
|
||||
}
|
||||
var request structuredoutput.PreparedSnapshot
|
||||
readInteraction(t, attempt.Artifacts.RequestPayloadPath, &request)
|
||||
if !reflect.DeepEqual(&request, captured[i]) {
|
||||
t.Fatalf("request %d did not preserve prepared evidence", i+1)
|
||||
}
|
||||
}
|
||||
attempt := later.Repairs[0].Attempts[1]
|
||||
if attempt.Revision == nil || attempt.Revision.ProposalIndex != 1 ||
|
||||
len(attempt.Decisions) != 1 || attempt.Decisions[0].ProposalIndex != 1 {
|
||||
t.Fatalf("lost ordered repair evidence: %+v", attempt)
|
||||
|
||||
Reference in New Issue
Block a user