Share repair histories while debug artifacts wait for indices
This commit is contained in:
@@ -142,6 +142,7 @@ func runRepairBranch(ctx context.Context, branchInput repairBranchInput) (repair
|
||||
promptID := branchInput.promptID
|
||||
current := original
|
||||
findings := repairFindings(original.ProposalIndex, branchInput.rejected)
|
||||
var history repairHistory
|
||||
buildRequest := func() (*structuredoutput.Request, error) {
|
||||
request := proposal_generation.RepairRequest{
|
||||
ProposalRef: proposalRef,
|
||||
@@ -192,6 +193,8 @@ func runRepairBranch(ctx context.Context, branchInput repairBranchInput) (repair
|
||||
record.Revision = &snapshot
|
||||
}
|
||||
feedback := findings
|
||||
prepared := history.retain(completion.Prepared)
|
||||
prompt, finalContent := completion.Prompt, completion.FinalContent
|
||||
record.Artifacts, _ = structuredoutput.WriteIndexedInteraction(input.Interactions, func(offset int, resolvePath func(string) string) structuredoutput.Interaction {
|
||||
index := proposalIndex + offset
|
||||
name := fmt.Sprintf("%s:section-%04d:proposal-%04d:attempt-%04d:repair", input.Spec.InstanceName, section, index, attemptNumber)
|
||||
@@ -217,9 +220,9 @@ func runRepairBranch(ctx context.Context, branchInput repairBranchInput) (repair
|
||||
return structuredoutput.Interaction{
|
||||
Name: name,
|
||||
Context: structuredoutput.DiagnosticContext{StageIndex: &input.Spec.Index, StageID: input.Spec.ID, Role: "correction_repair", ModuleKey: input.Spec.ModuleKey, ModuleInstance: input.Spec.InstanceName, SectionIndex: input.SectionIndex, ProposalIndex: &index, RepairAttempt: &attemptNumber},
|
||||
RequestMetadata: map[string]any{"stage_index": input.Spec.Index, "stage_id": input.Spec.ID, "proposal_index": index, "repair_attempt": attemptNumber, "prompt_id": promptID, "prompt": completion.Prompt, "feedback": feedback},
|
||||
RequestPayload: completion.Prepared,
|
||||
ResponsePayload: map[string]any{"decoded": response, "final_content": completion.FinalContent, "revision": correction, "decisions": decisions, "rejections": rejections},
|
||||
RequestMetadata: map[string]any{"stage_index": input.Spec.Index, "stage_id": input.Spec.ID, "proposal_index": index, "repair_attempt": attemptNumber, "prompt_id": promptID, "prompt": prompt, "feedback": feedback},
|
||||
RequestPayload: prepared,
|
||||
ResponsePayload: map[string]any{"decoded": response, "final_content": finalContent, "revision": correction, "decisions": decisions, "rejections": rejections},
|
||||
ErrorPayload: failure,
|
||||
}
|
||||
})
|
||||
|
||||
@@ -28,10 +28,11 @@ import (
|
||||
)
|
||||
|
||||
type repairClient struct {
|
||||
t *testing.T
|
||||
outputs []string
|
||||
calls []structuredoutput.Request
|
||||
err error
|
||||
t *testing.T
|
||||
outputs []string
|
||||
calls []structuredoutput.Request
|
||||
err error
|
||||
capturePrepared func(*structuredoutput.PreparedSnapshot)
|
||||
}
|
||||
|
||||
func TestRepairBranchesRetargetAndPreserveOtherApprovals(t *testing.T) {
|
||||
@@ -474,6 +475,9 @@ func (c *repairClient) Coordinate(ctx context.Context, _ *structuredoutput.Excha
|
||||
return false, err
|
||||
}
|
||||
return client.Coordinate(ctx, base.Exchange, budget, func(ctx context.Context, turn structuredoutput.Turn) (*structuredoutput.Request, error) {
|
||||
if turn.Response != nil && c.capturePrepared != nil {
|
||||
c.capturePrepared(turn.Response.Prepared)
|
||||
}
|
||||
req, err := next(ctx, turn)
|
||||
if req != nil && err == nil {
|
||||
selected = req
|
||||
|
||||
@@ -28,13 +28,46 @@ func (s *sectionInteractions) WriteIndexedInteraction(build structuredoutput.Ind
|
||||
}
|
||||
|
||||
func (s *sectionInteractions) write(offset int) {
|
||||
for _, build := range s.builders {
|
||||
for i, build := range s.builders {
|
||||
paths, _ := s.InteractionSink.WriteInteraction(build(offset, s.responsePath))
|
||||
s.written = append(s.written, paths)
|
||||
s.builders[i] = nil
|
||||
}
|
||||
s.builders = nil
|
||||
}
|
||||
|
||||
// repairHistory shares the append-only prefix of prepared repair requests.
|
||||
// Earlier snapshots keep their own lengths while additional turns add messages.
|
||||
type repairHistory struct {
|
||||
messages []structuredoutput.Message
|
||||
}
|
||||
|
||||
func (h *repairHistory) retain(prepared *structuredoutput.PreparedSnapshot) *structuredoutput.PreparedSnapshot {
|
||||
if prepared == nil {
|
||||
return nil
|
||||
}
|
||||
messages := prepared.Messages
|
||||
n := len(h.messages)
|
||||
if n == 0 || n > len(messages) || !slices.EqualFunc(h.messages, messages[:n], sameMessage) {
|
||||
h.messages = messages
|
||||
} else {
|
||||
h.messages = append(h.messages, messages[n:]...)
|
||||
}
|
||||
snapshot := *prepared
|
||||
snapshot.Messages = slices.Clip(h.messages)
|
||||
return &snapshot
|
||||
}
|
||||
|
||||
func sameMessage(a, b structuredoutput.Message) bool {
|
||||
if a.Role != b.Role || a.Content != b.Content {
|
||||
return false
|
||||
}
|
||||
if a.CacheControl == nil || b.CacheControl == nil {
|
||||
return a.CacheControl == b.CacheControl
|
||||
}
|
||||
return *a.CacheControl == *b.CacheControl
|
||||
}
|
||||
|
||||
func (s *sectionInteractions) artifacts(paths structuredoutput.InteractionArtifacts) structuredoutput.InteractionArtifacts {
|
||||
value, pending := strings.CutPrefix(paths.ResponsePayloadPath, pendingInteractionPrefix)
|
||||
if !pending {
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
@@ -194,10 +195,11 @@ func TestLaterRepairArtifactsUseOrderedProposalIndices(t *testing.T) {
|
||||
cancel()
|
||||
return contracts.StructuredCompletionResponse{}, ctx.Err()
|
||||
}
|
||||
lastRevision := revision && *req.Context.RepairAttempt == 2
|
||||
dst.(*validators.LLMValidationResponse).Validations = []validators.LLMValidationDecision{{
|
||||
CorrectionIndex: "1", Approved: first || revision, Reason: "checked",
|
||||
CorrectionIndex: "1", Approved: first || lastRevision, Reason: "checked",
|
||||
}}
|
||||
if revision {
|
||||
if lastRevision {
|
||||
close(laterRevalidated)
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{}, nil
|
||||
@@ -211,18 +213,30 @@ func TestLaterRepairArtifactsUseOrderedProposalIndices(t *testing.T) {
|
||||
started := make(chan struct{})
|
||||
close(started)
|
||||
module.headEntered = started
|
||||
repair := &repairClient{t: t, outputs: []string{
|
||||
var captured []*structuredoutput.PreparedSnapshot
|
||||
repair := &repairClient{t: t, capturePrepared: func(p *structuredoutput.PreparedSnapshot) {
|
||||
captured = append(captured, p)
|
||||
}, outputs: []string{
|
||||
`{"proposal_ref":1,"decision":{"action":"revise","correction":{"id":1,"original_text":"bravo","corrected_text":"BRAVO"}}}`,
|
||||
`{"proposal_ref":1,"decision":{"action":"revise","correction":{"id":1,"original_text":"bravo","corrected_text":"Bravo"}}}`,
|
||||
}}
|
||||
dir := t.TempDir()
|
||||
if canceled {
|
||||
blocked := filepath.Join(dir, "grammar", "grammar_section-0001_proposal-0001_attempt-0001_repair-request-payload.json")
|
||||
if err := os.MkdirAll(blocked, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
writer := llm.NewDiagnosticsWriter(dir)
|
||||
cfg := runnerTestConfig()
|
||||
cfg.Chunking.MaxSectionTokens = 1
|
||||
cfg.Chunking.MinSectionTokens = 0
|
||||
out, err := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}}).Run(ctx, RunInput{
|
||||
Config: &cfg, Interactions: llm.NewDiagnosticsWriter(t.TempDir()), Evidence: &EvidenceSelection{Reports: true, Debug: true},
|
||||
Config: &cfg, Interactions: writer, Evidence: &EvidenceSelection{Reports: true, Debug: true},
|
||||
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "alpha"}, {ID: 93, Text: "bravo"}}},
|
||||
ModuleSpecs: []contracts.ModuleRunSpec{{
|
||||
ModuleKey: "grammar", InstanceName: "grammar", ProposalClient: proposal, ValidationClient: validation,
|
||||
RepairCoordinator: repair, Settings: config.Stage{Validation: cfg.Validation, LLM: config.LLM{CorrectionRepairAttempts: 1}},
|
||||
RepairCoordinator: repair, Settings: config.Stage{Validation: cfg.Validation, LLM: config.LLM{CorrectionRepairAttempts: 2}},
|
||||
}},
|
||||
})
|
||||
if (!canceled && err != nil) || (canceled && !errors.Is(err, context.Canceled)) {
|
||||
@@ -230,10 +244,26 @@ func TestLaterRepairArtifactsUseOrderedProposalIndices(t *testing.T) {
|
||||
}
|
||||
later := out.WorkUnits[1]
|
||||
if len(out.Candidates) != 2 || out.Candidates[1].ProposalIndex != 1 ||
|
||||
len(later.Repairs) != 1 || later.Repairs[0].ProposalIndex != 1 || len(later.Repairs[0].Attempts) != 1 {
|
||||
len(later.Repairs) != 1 || later.Repairs[0].ProposalIndex != 1 || len(later.Repairs[0].Attempts) != 2 {
|
||||
t.Fatalf("lost ordered candidate or repair: %+v, %+v", out.Candidates, later.Repairs)
|
||||
}
|
||||
attempt := later.Repairs[0].Attempts[0]
|
||||
if len(captured) != 2 || len(captured[1].Messages) <= len(captured[0].Messages) {
|
||||
t.Fatalf("missing growing repair history: %+v", captured)
|
||||
}
|
||||
for i, attempt := range later.Repairs[0].Attempts {
|
||||
if i == 0 && canceled {
|
||||
if attempt.Artifacts.RequestPayloadPath != "" || len(writer.Warnings()) != 1 {
|
||||
t.Fatalf("expected one optional request write failure: %+v, %v", attempt.Artifacts, writer.Warnings())
|
||||
}
|
||||
continue
|
||||
}
|
||||
var request structuredoutput.PreparedSnapshot
|
||||
readInteraction(t, attempt.Artifacts.RequestPayloadPath, &request)
|
||||
if !reflect.DeepEqual(&request, captured[i]) {
|
||||
t.Fatalf("request %d did not preserve prepared evidence", i+1)
|
||||
}
|
||||
}
|
||||
attempt := later.Repairs[0].Attempts[1]
|
||||
if attempt.Revision == nil || attempt.Revision.ProposalIndex != 1 ||
|
||||
len(attempt.Decisions) != 1 || attempt.Decisions[0].ProposalIndex != 1 {
|
||||
t.Fatalf("lost ordered repair evidence: %+v", attempt)
|
||||
|
||||
Reference in New Issue
Block a user