Files
audita/internal/framework/runner/repair_test.go

493 lines
29 KiB
Go

package runner
import (
"context"
"encoding/json"
"errors"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
ck "go.maximumdirect.net/configkit"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
)
type repairClient struct {
t *testing.T
outputs []string
calls []structuredoutput.Request
err error
}
func TestRepairBranchesRetargetAndPreserveOtherApprovals(t *testing.T) {
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":2,"decision":{"action":"revise","correction":{"id":2,"original_text":"uh ","corrected_text":""}}}`}}
candidates := []proposals.EnrichedCorrectionProposal{
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 4}},
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 0, OriginalText: "x", CorrectedText: "y"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 5}},
}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", RepairCoordinator: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}, {ID: 2, Text: "uh hello"}}}, Validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.NonEmptyResultingSegmentValidator{}}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{}}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
}
result, branches, err := repairCandidates(context.Background(), input, []int{1, 9}, initial, "repair")
if err != nil {
t.Fatal(err)
}
if len(result.approved) != 2 || result.approved[1].ProposalIndex != 5 || result.approved[1].TargetSegmentID != 2 || result.approved[1].CorrectedText != "" || len(branches) != 1 || branches[0].Outcome != "approved" || len(result.filtered) != 0 {
t.Fatalf("result=%+v branches=%+v", result, branches)
}
var payload proposal_generation.RepairRequest
if err := json.Unmarshal([]byte(client.calls[0].Variables["RepairRequestJSON"]), &payload); err != nil {
t.Fatal(err)
}
if payload.ProposalRef != 2 || payload.Correction.SegmentReference != 9 || payload.OriginalSegmentText != nil || payload.CorrectedSegmentText != nil {
t.Fatalf("request: %+v", payload)
}
}
func TestRepairRequestCarriesLiteralPreview(t *testing.T) {
for _, tc := range []struct {
name, segment, original, replacement, result string
policy proposals.ReplacementPolicy
preview bool
}{
{"deletion", "uh hello", "uh ", "", "hello", proposals.ReplacementPolicyRequireUnique, true},
{"all occurrences", "gestures and gestures", "gestures", "Jesters", "Jesters and Jesters", proposals.ReplacementPolicyReplaceAll, true},
{"ambiguous span", "gestures and gestures", "gestures", "Jesters", "", proposals.ReplacementPolicyRequireUnique, false},
{"missing span", "hello", "absent", "world", "", proposals.ReplacementPolicyRequireUnique, false},
{"blank result", "uh", "uh", "", "", proposals.ReplacementPolicyRequireUnique, false},
} {
t.Run(tc.name, func(t *testing.T) {
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`}}
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 81, OriginalText: tc.original, CorrectedText: tc.replacement}}
input := validateSectionCandidatesInput{
Spec: contracts.ModuleRunSpec{RepairCoordinator: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}},
Policy: tc.policy,
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 81, Text: tc.segment}}},
Validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.NonEmptyResultingSegmentValidator{}, rejectUntilRepair()},
SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate},
Exchange: &structuredoutput.Exchange{},
}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
}
_, branches, err := repairCandidates(context.Background(), input, []int{1}, initial, "repair")
if err != nil || len(client.calls) != 1 || len(branches) != 1 || branches[0].Outcome != "withdrawn" {
t.Fatalf("repair: branches=%+v calls=%d err=%v", branches, len(client.calls), err)
}
var payload proposal_generation.RepairRequest
if err := json.Unmarshal([]byte(client.calls[0].Variables["RepairRequestJSON"]), &payload); err != nil {
t.Fatal(err)
}
if payload.OriginalSegmentText == nil || *payload.OriginalSegmentText != tc.segment || len(payload.Findings) == 0 {
t.Fatalf("missing source or findings: %+v", payload)
}
if tc.preview {
if payload.CorrectedSegmentText == nil || *payload.CorrectedSegmentText != tc.result {
t.Fatalf("incorrect preview: %+v", payload)
}
} else if payload.CorrectedSegmentText != nil {
t.Fatalf("inapplicable proposal has a preview: %+v", payload)
}
if input.WorkingTranscript.Segments[0].Text != tc.segment {
t.Fatal("repair preview changed the transcript")
}
})
}
}
func TestRepairRequestRefreshesPreviewAfterRetarget(t *testing.T) {
client := &repairClient{t: t, outputs: []string{
`{"proposal_ref":1,"decision":{"action":"revise","correction":{"id":2,"original_text":"uh ","corrected_text":""}}}`,
`{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`,
}}
var validatedIDs []int
reject := fakeValidator{name: "editorial_review", validateF: func(req validators.Request) (validators.Result, error) {
validatedIDs = append(validatedIDs, req.CandidateProposal[0].TargetSegmentID)
return validators.Result{Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Disposition: validators.DispositionRejected, ReasonCode: "llm_rejected", Message: "The context does not support this edit."}}}, nil
}}
input := validateSectionCandidatesInput{
Spec: contracts.ModuleRunSpec{RepairCoordinator: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 2}}},
Policy: proposals.ReplacementPolicyRequireUnique,
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 81, Text: "cat"}, {ID: 104, Text: "uh hello"}}},
Validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validatormetadata.Wrap(reject, validatormetadata.ExecutionClassDeterministic)},
SectionEnriched: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 81, OriginalText: "cat", CorrectedText: "dog"}}},
Exchange: &structuredoutput.Exchange{},
Interactions: llm.NewDiagnosticsWriter(t.TempDir()),
}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
}
_, branches, err := repairCandidates(context.Background(), input, []int{1}, initial, "repair")
if err != nil || len(client.calls) != 2 || len(branches) != 1 || branches[0].Outcome != "withdrawn" {
t.Fatalf("repair: branches=%+v calls=%d err=%v", branches, len(client.calls), err)
}
if len(validatedIDs) != 2 || validatedIDs[0] != 81 || validatedIDs[1] != 104 {
t.Fatalf("validated segment IDs: %v", validatedIDs)
}
for _, decisions := range [][]ValidatorDecisionRecord{initial.decisions, branches[0].Attempts[0].Decisions} {
if len(decisions) == 0 {
t.Fatal("missing decision evidence")
}
record := decisions[len(decisions)-1]
if record.ValidatorName != "editorial_review" || record.ExecutionClass != validatormetadata.ExecutionClassDeterministic {
t.Fatalf("lost execution class override: %+v", record)
}
}
for i, want := range []struct{ before, after string }{{"cat", "dog"}, {"uh hello", "hello"}} {
var payload proposal_generation.RepairRequest
if err := json.Unmarshal([]byte(client.calls[i].Variables["RepairRequestJSON"]), &payload); err != nil {
t.Fatal(err)
}
if payload.ProposalRef != 1 || payload.Correction.SegmentReference != i+1 || payload.OriginalSegmentText == nil || *payload.OriginalSegmentText != want.before || payload.CorrectedSegmentText == nil || *payload.CorrectedSegmentText != want.after {
t.Fatalf("attempt %d did not describe its current proposal: %+v", i+1, payload)
}
}
}
func TestRepairBranchTerminalOutcomes(t *testing.T) {
for _, tc := range []struct {
name string
outputs []string
err error
budget int
outcome string
calls int
}{
{"withdraw", []string{`{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`}, nil, 1, "withdrawn", 1},
{"exhausted", []string{`{"proposal_ref":1,"decision":{"action":"revise","correction":{"id":1,"original_text":"absent","corrected_text":"dog"}}}`, `{"proposal_ref":1,"decision":{"action":"revise","correction":{"id":1,"original_text":"absent","corrected_text":"dog"}}}`}, nil, 2, "exhausted", 2},
{"no-effect revision", []string{`{"proposal_ref":1,"decision":{"action":"revise","correction":{"id":1,"original_text":"cat","corrected_text":"cat"}}}`}, nil, 2, "rejected", 1},
{"wrong reference", []string{`{"proposal_ref":2,"decision":{"action":"withdraw","correction":null}}`}, nil, 1, "unavailable", 1},
{"provider failure", nil, &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonProviderFailed, Message: "provider failed", Cause: errors.New("underlying provider error")}, 1, "unavailable", 1},
} {
t.Run(tc.name, func(t *testing.T) {
client := &repairClient{t: t, outputs: tc.outputs, err: tc.err}
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairCoordinator: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: tc.budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
}
result, branches, err := repairCandidates(context.Background(), input, []int{1}, initial, "repair")
if err != nil {
t.Fatal(err)
}
if len(branches) != 1 || branches[0].Outcome != tc.outcome || len(client.calls) != tc.calls || branches[0].AttemptCount != tc.calls || len(branches[0].Attempts) != 0 {
t.Fatalf("branches=%+v calls=%d", branches, len(client.calls))
}
if tc.err != nil {
operation := result.operations[len(result.operations)-1]
if operation.FailureClass != outcomes.FailureRecoverable ||
operation.FailureReason != outcomes.ReasonProviderFailed || !errors.Is(operation.Cause, tc.err) {
t.Fatalf("repair lost typed failure facts: %+v", operation)
}
}
if tc.calls == 2 && client.calls[1].Context.RepairAttempt == nil {
t.Fatalf("second attempt lacks diagnostic coordinates: %+v", client.calls[1].Context)
}
})
}
}
type repairModule struct{ outcomeModule }
func (m repairModule) RepairPromptID() string { return "repair" }
func TestRunnerChecksRepairCapabilityBeforeSubmittingProposals(t *testing.T) {
calls := 0
module := repairModule{outcomeModule{fakeModule: fakeModule{key: "grammar"}, proposeResult: func(contracts.ProposalRequest) (contracts.ProposalResult, error) {
calls++
return contracts.ProposalResult{}, nil
}}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}})
out, err := r.Run(context.Background(), RunInput{
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{
ModuleKey: "grammar", InstanceName: "grammar",
Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}},
}},
})
var failure *structuredoutput.Failure
if !errors.As(err, &failure) || failure.Class != outcomes.FailureFatal || failure.Reason != outcomes.ReasonSetupFailure ||
calls != 0 || len(out.WorkUnits) != 1 || out.WorkUnits[0].State != outcomes.WorkNotStarted {
t.Fatalf("repair setup did not stop before dispatch: calls=%d work=%+v error=%v", calls, out.WorkUnits, err)
}
}
func TestRunnerAppliesRevisionsInOriginalOrder(t *testing.T) {
candidates := []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, {TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "fox"}, {TargetSegmentID: 1, OriginalText: "bird", CorrectedText: "owl"}}
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":2,"decision":{"action":"revise","correction":{"id":1,"original_text":"cat","corrected_text":"fox"}}}`,
`{"proposal_ref":3,"decision":{"action":"revise","correction":{"id":1,"original_text":"bird","corrected_text":"owl"}}}`,
}}
module := repairModule{outcomeModule{fakeModule: fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, rejectUntilRepair(0)}}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
return contracts.ProposalResult{Proposals: candidates, SegmentReferences: []int{1, 1, 1}, Exchange: &structuredoutput.Exchange{}}, nil
}}}
runner := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}})
out, err := runner.Run(context.Background(), RunInput{Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat and bird"}}}, ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", RepairCoordinator: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}}})
if err != nil {
t.Fatal(err)
}
if out.FinalTranscript.Segments[0].Text != "dog and owl" || len(out.WorkUnits) != 1 || len(out.WorkUnits[0].Repairs) != 2 || out.WorkUnits[0].Repairs[0].Outcome != "approved_skipped" || out.WorkUnits[0].Repairs[1].Outcome != "applied" || out.WorkUnits[0].CorrectionsApplied != 2 || len(out.ModuleResults[0].SkippedChanges) != 1 || out.ModuleResults[0].SkippedChanges[0].ProposalIndex != 1 {
t.Fatalf("output=%+v", out)
}
operations := out.WorkUnits[0].Operations
if len(operations) != 2 {
t.Fatalf("repair operations: %+v", operations)
}
for _, op := range operations {
if op.Reference.Kind != outcomes.OperationCorrectionRepair || len(op.RecoveryReasons) != 1 || op.RecoveryReasons[0] != outcomes.RecoveryCorrectionRevised {
t.Fatalf("repair operation: %+v", op)
}
}
}
func TestRepairRevalidatesEarlierApprovalAndRetainsUnrelatedApprovalOnFailure(t *testing.T) {
candidates := []proposals.EnrichedCorrectionProposal{
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}},
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "bird", CorrectedText: "owl"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}},
}
for _, tc := range []struct {
name string
failure bool
}{{"revision", false}, {"provider failure", true}} {
t.Run(tc.name, func(t *testing.T) {
calls := 0
check := fakeValidator{name: "custom", validateF: func(req validators.Request) (validators.Result, error) {
var decisions []validators.Decision
for _, candidate := range req.CandidateProposal {
calls++
decisions = append(decisions, validators.Decision{ProposalIndex: candidate.ProposalIndex, Approved: true, Disposition: validators.DispositionApproved})
}
return validators.Result{Decisions: decisions}, nil
}}
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":2,"decision":{"action":"revise","correction":{"id":1,"original_text":"bird","corrected_text":"owl"}}}`}}
if tc.failure {
client.err = &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonProviderFailed, Message: "failed"}
}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairCoordinator: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat bird"}}}, Validators: []validators.Validator{check, rejectUntilRepair(0)}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{}}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
}
got, branches, err := repairCandidates(context.Background(), input, []int{1, 1}, initial, "repair")
if err != nil {
t.Fatal(err)
}
if tc.failure {
if len(got.approved) != 1 || got.approved[0].ProposalIndex != 0 || branches[0].Outcome != "unavailable" || calls != 2 {
t.Fatalf("result=%+v calls=%d", got, calls)
}
} else if len(got.approved) != 2 || branches[0].Outcome != "approved" || calls != 3 {
t.Fatalf("result=%+v calls=%d", got, calls)
}
})
}
}
func TestRepairInterruptionRetainsAttempt(t *testing.T) {
client := &repairClient{t: t, err: &structuredoutput.Failure{Class: structuredoutput.FailureCanceled, Reason: outcomes.ReasonCanceled, Message: "canceled", Cause: context.Canceled}}
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairCoordinator: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
}
_, branches, err := repairCandidates(context.Background(), input, []int{1}, initial, "repair")
if !errors.Is(err, context.Canceled) || len(branches) != 1 || branches[0].Outcome != "interrupted" || branches[0].AttemptCount != 1 || len(branches[0].Attempts) != 0 {
t.Fatalf("branches=%+v error=%v", branches, err)
}
}
func TestRepairArtifactsAreUnique(t *testing.T) {
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`, `{"proposal_ref":2,"decision":{"action":"withdraw","correction":null}}`}}
candidates := []proposals.EnrichedCorrectionProposal{
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}},
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "fox"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}},
}
section := 4
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar_1", RepairCoordinator: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{rejectUntilRepair()}, SectionEnriched: candidates, SectionIndex: &section, Exchange: &structuredoutput.Exchange{}, Interactions: llm.NewDiagnosticsWriter(t.TempDir())}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
}
_, branches, err := repairCandidates(context.Background(), input, []int{1, 1}, initial, "repair")
if err != nil {
t.Fatal(err)
}
if len(branches) != 2 {
t.Fatalf("branches: %+v", branches)
}
a, b := branches[0].Attempts[0].Artifacts.RequestPayloadPath, branches[1].Attempts[0].Artifacts.RequestPayloadPath
if a == "" || a == b {
t.Fatalf("artifact paths: %q %q", a, b)
}
for i, path := range []string{a, b} {
data, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
if !json.Valid(data) {
t.Fatalf("invalid JSON artifact %s: %s", filepath.Base(path), data)
}
raw, err := os.ReadFile(branches[i].Attempts[0].Artifacts.RequestMetadataPath)
if err != nil {
t.Fatal(err)
}
var metadata struct {
Context structuredoutput.DiagnosticContext `json:"context"`
}
if err := json.Unmarshal(raw, &metadata); err != nil {
t.Fatal(err)
}
c := metadata.Context
if c.Role != "correction_repair" || c.ModuleInstance != "grammar_1" || c.SectionIndex == nil || *c.SectionIndex != section ||
c.ProposalIndex == nil || *c.ProposalIndex != i || c.RepairAttempt == nil || *c.RepairAttempt != 1 {
t.Fatalf("repair metadata: %s", raw)
}
var request proposal_generation.RepairRequest
if err := json.Unmarshal([]byte(client.calls[i].Variables["RepairRequestJSON"]), &request); err != nil {
t.Fatal(err)
}
if request.ProposalRef != i+1 || request.Correction.CorrectedText != candidates[i].CorrectedText ||
len(request.Findings) != 1 || request.Findings[0].Explanation != "The proposed replacement is unsupported by the supplied context." {
t.Fatalf("repair request for proposal %d: %+v", i, request)
}
}
}
func TestRepairArtifactFailureDoesNotChangeOutcome(t *testing.T) {
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`}}
path := filepath.Join(t.TempDir(), "file")
if err := os.WriteFile(path, []byte("x"), 0600); err != nil {
t.Fatal(err)
}
sink := llm.NewDiagnosticsWriter(path)
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairCoordinator: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}, Interactions: sink}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
}
_, branches, err := repairCandidates(context.Background(), input, []int{1}, initial, "repair")
if err != nil || len(branches) != 1 || branches[0].Outcome != "withdrawn" || len(sink.Warnings()) != 1 {
t.Fatalf("branches=%+v warnings=%v err=%v", branches, sink.Warnings(), err)
}
}
func TestNoEffectAndUnavailableOnlyDoNotStartRepair(t *testing.T) {
client := &repairClient{t: t}
for _, rejected := range [][]ValidatorRejectedChange{
{{ProposalIndex: 0, Disposition: validators.DispositionRejected, ReasonCode: validators.ReasonNoEffect}},
{{ProposalIndex: 0, Disposition: validators.DispositionUnavailable, ReasonCode: "provider_failed"}},
} {
candidate := proposals.EnrichedCorrectionProposal{ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairCoordinator: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}}
_, branches, err := repairCandidates(context.Background(), input, nil, validateSectionCandidatesResult{rejected: rejected}, "repair")
if err != nil || len(branches) != 0 || len(client.calls) != 0 {
t.Fatalf("branches=%+v calls=%d err=%v", branches, len(client.calls), err)
}
}
}
// rejectUntilRepair supplies a semantic rejection at the external validator seam.
func rejectUntilRepair(preserved ...int) validators.Validator {
return fakeValidator{name: "editorial_review", validateF: func(req validators.Request) (validators.Result, error) {
var decisions []validators.Decision
for _, p := range req.CandidateProposal {
approved := req.RepairAttempt != nil
for _, index := range preserved {
if p.ProposalIndex == index {
approved = true
}
}
disposition, reason := validators.DispositionRejected, "llm_rejected"
if approved {
disposition, reason = validators.DispositionApproved, validators.ReasonApproved
}
decisions = append(decisions, validators.Decision{ProposalIndex: p.ProposalIndex, Approved: approved, Disposition: disposition, ReasonCode: reason, Message: "The proposed replacement is unsupported by the supplied context."})
}
return validators.Result{Decisions: decisions}, nil
}}
}
type repairTransport func(*http.Request) (*http.Response, error)
func (f repairTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
// Exercise the real adapter and upstream coordinator; only model responses are scripted.
func (c *repairClient) Coordinate(ctx context.Context, _ *structuredoutput.Exchange, budget int, next func(context.Context, structuredoutput.Turn) (*structuredoutput.Request, error)) (bool, error) {
t := c.t
dir := t.TempDir()
for name, content := range map[string]string{
"config.yml": "version: 3\n",
"prompts.yml": "profiles:\n - id: fixture\n endpoint: https://example.invalid/v1\n model: fixture\nprompts:\n - id: repair\n messages:\n - role: user\n content: '{{.RepairRequestJSON}}'\n output:\n format: text\n validation_mode: none\n",
} {
if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0600); err != nil {
t.Fatal(err)
}
}
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
if err != nil {
t.Fatal(err)
}
var selected *structuredoutput.Request
initial := true
runtime, err := llm.NewRuntime(ctx, loaded, &http.Client{Transport: repairTransport(func(*http.Request) (*http.Response, error) {
content := `{"corrections":[]}`
if initial {
initial = false
} else {
c.calls = append(c.calls, *selected)
content = c.outputs[len(c.calls)-1]
}
body, _ := json.Marshal(map[string]any{"choices": []any{map[string]any{"message": map[string]any{"role": "assistant", "content": content}, "finish_reason": "stop"}}})
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(string(body)))}, nil
})})
if err != nil {
t.Fatal(err)
}
client, err := runtime.NewRoleClient("repair", "fixture", config.LLM{OperationTimeout: time.Second}, llm.CallHooks{})
if err != nil {
t.Fatal(err)
}
var output map[string]any
base, err := client.CompleteStructured(ctx, structuredoutput.Request{PromptID: prompts.PromptIDModuleGrammarProposal, Schema: responseschema.CorrectionSetKey, CaptureExchange: true, Variables: map[string]string{"TranscriptDescriptionBlock": "", "GlossaryJSON": "[]", "SectionJSON": "[]"}}, &output)
if err != nil {
return false, err
}
return client.Coordinate(ctx, base.Exchange, budget, func(ctx context.Context, turn structuredoutput.Turn) (*structuredoutput.Request, error) {
req, err := next(ctx, turn)
if req != nil && err == nil {
selected = req
if c.err != nil {
c.calls = append(c.calls, *req)
_, err = next(ctx, structuredoutput.Turn{Response: &structuredoutput.Response{}, Err: c.err})
return nil, err
}
// Test prompt IDs vary; all use the same supplied-data fixture definition.
copy := *req
copy.PromptID = "repair"
return &copy, nil
}
return req, err
})
}