Validate internal decisions and reconcile exact model references
This commit is contained in:
@@ -104,7 +104,7 @@ func TestValidatorFailureCancelsActiveAndPendingProviderWork(t *testing.T) {
|
||||
}, &out)
|
||||
return validators.Result{}, err
|
||||
}}
|
||||
module := fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{validator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
module := fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{validator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "hello", CorrectedText: "Hello"}}, nil
|
||||
}}
|
||||
sections := []chunking.Section{}
|
||||
|
||||
@@ -1,48 +1,19 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
||||
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
|
||||
)
|
||||
|
||||
// feedbackCandidates submits only candidates for which the validator has usable evidence.
|
||||
func feedbackCandidates(input validateSectionCandidatesInput, validator contracts.Validator, noEffect map[int]bool, evidence contracts.ValidationRequest) []proposals.EnrichedCorrectionProposal {
|
||||
func feedbackCandidates(input validateSectionCandidatesInput, validatorName string, modelDriven bool, noEffect map[int]bool, evidence validators.Request) []proposals.EnrichedCorrectionProposal {
|
||||
var result []proposals.EnrichedCorrectionProposal
|
||||
for _, candidate := range input.SectionEnriched {
|
||||
if noEffect[candidate.ProposalIndex] {
|
||||
continue
|
||||
}
|
||||
usableSpan := candidate.OriginalText != ""
|
||||
target := candidate.TargetSegmentID > 0
|
||||
var applicable bool
|
||||
switch validator.Name() {
|
||||
case "proposal_shape":
|
||||
applicable = true
|
||||
case "no_effect":
|
||||
applicable = usableSpan
|
||||
case "original_text_presence":
|
||||
applicable = usableSpan && target
|
||||
case "protected_terms":
|
||||
applicable = usableSpan
|
||||
if input.Spec.ModuleKey == modulecatalog.KeyGlossary || input.Spec.ModuleKey == modulecatalog.KeySpokenWord {
|
||||
preview := evidence.Preview(candidate)
|
||||
applicable = preview.Applicable || preview.SkipReason == proposals.SkipReasonEmptyResultingText
|
||||
}
|
||||
case "non_empty_resulting_segment":
|
||||
preview := evidence.Preview(candidate)
|
||||
applicable = preview.Applicable || preview.SkipReason == proposals.SkipReasonEmptyResultingText
|
||||
default:
|
||||
if validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked {
|
||||
applicable = evidence.Preview(candidate).Applicable
|
||||
} else {
|
||||
applicable = true
|
||||
}
|
||||
}
|
||||
if applicable {
|
||||
if evidence.AcceptsIndependentCandidate(validatorName, modelDriven, candidate) {
|
||||
result = append(result, candidate)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,7 +25,7 @@ func TestFeedbackCollectsIndependentFindingsAndUsableSemanticInput(t *testing.T)
|
||||
return proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: target, OriginalText: original, CorrectedText: replacement}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index}}
|
||||
}
|
||||
var seen []int
|
||||
semantic := validatormetadata.Wrap(fakeValidator{name: "semantic", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
semantic := validatormetadata.Wrap(fakeValidator{name: "semantic", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
var decisions []validators.Decision
|
||||
for _, p := range req.CandidateProposal {
|
||||
seen = append(seen, p.ProposalIndex)
|
||||
@@ -33,7 +33,7 @@ func TestFeedbackCollectsIndependentFindingsAndUsableSemanticInput(t *testing.T)
|
||||
}
|
||||
return validators.Result{Decisions: decisions}, nil
|
||||
}}, validatormetadata.ExecutionClassLLMBacked)
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh teh cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, semantic}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate(0, 1, "cat", "dog"), candidate(1, 0, "teh", "the"), candidate(2, 1, "teh", "the"), candidate(3, 1, "cat", "cat")}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh teh cat"}}}, Validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, semantic}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate(0, 1, "cat", "dog"), candidate(1, 0, "teh", "the"), candidate(2, 1, "teh", "the"), candidate(3, 1, "cat", "cat")}}
|
||||
result, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -58,14 +58,14 @@ func TestFeedbackKeepsUnavailableAndZeroBudgetShortlists(t *testing.T) {
|
||||
for _, budget := range []int{0, 1} {
|
||||
t.Run(fmt.Sprint(budget), func(t *testing.T) {
|
||||
var later int
|
||||
first := fakeValidator{name: "first", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
first := fakeValidator{name: "first", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
var decisions []validators.Decision
|
||||
for _, p := range req.CandidateProposal {
|
||||
decisions = append(decisions, validators.Decision{ProposalIndex: p.ProposalIndex, Disposition: validators.DispositionUnavailable, ReasonCode: "unavailable"})
|
||||
}
|
||||
return validators.Result{Decisions: decisions}, nil
|
||||
}}
|
||||
second := fakeValidator{name: "second", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
second := fakeValidator{name: "second", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
later = len(req.CandidateProposal)
|
||||
var decisions []validators.Decision
|
||||
for _, p := range req.CandidateProposal {
|
||||
@@ -73,7 +73,7 @@ func TestFeedbackKeepsUnavailableAndZeroBudgetShortlists(t *testing.T) {
|
||||
}
|
||||
return validators.Result{Decisions: decisions}, nil
|
||||
}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{first, second}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{first, second}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}}
|
||||
result, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -95,8 +95,8 @@ func TestRepairFindingsUseSafeUnavailableEvidence(t *testing.T) {
|
||||
|
||||
func TestRevalidationArtifactsUseProposalAndAttemptCoordinates(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{
|
||||
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}},
|
||||
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}},
|
||||
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}},
|
||||
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}},
|
||||
}}
|
||||
validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
|
||||
if err != nil {
|
||||
@@ -108,7 +108,7 @@ func TestRevalidationArtifactsUseProposalAndAttemptCoordinates(t *testing.T) {
|
||||
var paths []string
|
||||
for _, index := range []int{2, 3} {
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar_1", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validator}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, ValidationLLMClient: client, Interactions: sink, SectionIndex: §ion, RepairProposalIndex: &index, RepairAttempt: &attempt}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar_1", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{validator}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, ValidationLLMClient: client, Interactions: sink, SectionIndex: §ion, RepairProposalIndex: &index, RepairAttempt: &attempt}
|
||||
result, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -149,11 +149,11 @@ func TestRevalidationArtifactsUseProposalAndAttemptCoordinates(t *testing.T) {
|
||||
|
||||
func TestFeedbackReportsProtectionAndBlankResult(t *testing.T) {
|
||||
identity := glossary.Identity{Collection: "party", ID: "jesters"}
|
||||
protected := fakeValidator{name: "protected_terms", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
protected := fakeValidator{name: "protected_terms", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
return validators.ValidateProtectedTerms(req)
|
||||
}}
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "Jesters", CorrectedText: ""}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "spoken_word", ProtectedTerms: []glossary.Term{{Spelling: "Jesters", Identities: []glossary.Identity{identity}}}, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Jesters"}}}, Validators: []contracts.Validator{protected, validators.NonEmptyResultingSegmentValidator{}}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "spoken_word", ProtectedTerms: []glossary.Term{{Spelling: "Jesters", Identities: []glossary.Identity{identity}}}, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Jesters"}}}, Validators: []validators.Validator{protected, validators.NonEmptyResultingSegmentValidator{}}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}}
|
||||
result, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -2,6 +2,7 @@ package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||
@@ -27,7 +28,7 @@ func (m orderedModule) Key() string { return m.spec.ModuleKey }
|
||||
func (m orderedModule) ReplacementPolicy() proposals.ReplacementPolicy {
|
||||
return proposals.ReplacementPolicyRequireUnique
|
||||
}
|
||||
func (m orderedModule) Validators() []contracts.Validator { return nil }
|
||||
func (m orderedModule) Validators() []frameworkvalidators.Validator { return nil }
|
||||
func (m orderedModule) Propose(_ context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
m.seen <- req
|
||||
original, corrected := "party", "Party"
|
||||
|
||||
@@ -24,6 +24,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
||||
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
|
||||
)
|
||||
|
||||
type repairClient struct {
|
||||
@@ -52,7 +53,7 @@ func TestRepairBranchesRetargetAndPreserveOtherApprovals(t *testing.T) {
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 4}},
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 0, OriginalText: "x", CorrectedText: "y"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 5}},
|
||||
}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}, {ID: 2, Text: "uh hello"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.NonEmptyResultingSegmentValidator{}}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}, {ID: 2, Text: "uh hello"}}}, Validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.NonEmptyResultingSegmentValidator{}}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{}}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -92,7 +93,7 @@ func TestRepairRequestCarriesLiteralPreview(t *testing.T) {
|
||||
Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}},
|
||||
Policy: tc.policy,
|
||||
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 81, Text: tc.segment}}},
|
||||
Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.NonEmptyResultingSegmentValidator{}, rejectUntilRepair()},
|
||||
Validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.NonEmptyResultingSegmentValidator{}, rejectUntilRepair()},
|
||||
SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate},
|
||||
Exchange: &structuredoutput.Exchange{},
|
||||
}
|
||||
@@ -131,7 +132,7 @@ func TestRepairRequestRefreshesPreviewAfterRetarget(t *testing.T) {
|
||||
`{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`,
|
||||
}}
|
||||
var validatedIDs []int
|
||||
reject := fakeValidator{name: "editorial_review", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
reject := fakeValidator{name: "editorial_review", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
validatedIDs = append(validatedIDs, req.CandidateProposal[0].TargetSegmentID)
|
||||
return validators.Result{Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Disposition: validators.DispositionRejected, ReasonCode: "llm_rejected", Message: "The context does not support this edit."}}}, nil
|
||||
}}
|
||||
@@ -139,9 +140,10 @@ func TestRepairRequestRefreshesPreviewAfterRetarget(t *testing.T) {
|
||||
Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 2}}},
|
||||
Policy: proposals.ReplacementPolicyRequireUnique,
|
||||
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 81, Text: "cat"}, {ID: 104, Text: "uh hello"}}},
|
||||
Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, reject},
|
||||
Validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validatormetadata.Wrap(reject, validatormetadata.ExecutionClassDeterministic)},
|
||||
SectionEnriched: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 81, OriginalText: "cat", CorrectedText: "dog"}}},
|
||||
Exchange: &structuredoutput.Exchange{},
|
||||
Interactions: llm.NewDiagnosticsWriter(t.TempDir()),
|
||||
}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
@@ -154,6 +156,15 @@ func TestRepairRequestRefreshesPreviewAfterRetarget(t *testing.T) {
|
||||
if len(validatedIDs) != 2 || validatedIDs[0] != 81 || validatedIDs[1] != 104 {
|
||||
t.Fatalf("validated segment IDs: %v", validatedIDs)
|
||||
}
|
||||
for _, decisions := range [][]ValidatorDecisionRecord{initial.decisions, branches[0].Attempts[0].Decisions} {
|
||||
if len(decisions) == 0 {
|
||||
t.Fatal("missing decision evidence")
|
||||
}
|
||||
record := decisions[len(decisions)-1]
|
||||
if record.ValidatorName != "editorial_review" || record.ExecutionClass != validatormetadata.ExecutionClassDeterministic {
|
||||
t.Fatalf("lost execution class override: %+v", record)
|
||||
}
|
||||
}
|
||||
for i, want := range []struct{ before, after string }{{"cat", "dog"}, {"uh hello", "hello"}} {
|
||||
var payload proposal_generation.RepairRequest
|
||||
if err := json.Unmarshal([]byte(client.calls[i].Variables["RepairRequestJSON"]), &payload); err != nil {
|
||||
@@ -183,7 +194,7 @@ func TestRepairBranchTerminalOutcomes(t *testing.T) {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
client := &repairClient{t: t, outputs: tc.outputs, err: tc.err}
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: tc.budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: tc.budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -211,7 +222,7 @@ func TestRunnerAppliesRevisionsInOriginalOrder(t *testing.T) {
|
||||
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":2,"decision":{"action":"revise","correction":{"id":1,"original_text":"cat","corrected_text":"fox"}}}`,
|
||||
`{"proposal_ref":3,"decision":{"action":"revise","correction":{"id":1,"original_text":"bird","corrected_text":"owl"}}}`,
|
||||
}}
|
||||
module := repairModule{outcomeModule{fakeModule: fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, rejectUntilRepair(0)}}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
module := repairModule{outcomeModule{fakeModule: fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, rejectUntilRepair(0)}}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
return contracts.ProposalResult{Proposals: candidates, SegmentReferences: []int{1, 1, 1}, Exchange: &structuredoutput.Exchange{}}, nil
|
||||
}}}
|
||||
runner := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}})
|
||||
@@ -244,7 +255,7 @@ func TestRepairRevalidatesEarlierApprovalAndRetainsUnrelatedApprovalOnFailure(t
|
||||
}{{"revision", false}, {"provider failure", true}} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
calls := 0
|
||||
check := fakeValidator{name: "custom", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
check := fakeValidator{name: "custom", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
var decisions []validators.Decision
|
||||
for _, candidate := range req.CandidateProposal {
|
||||
calls++
|
||||
@@ -256,7 +267,7 @@ func TestRepairRevalidatesEarlierApprovalAndRetainsUnrelatedApprovalOnFailure(t
|
||||
if tc.failure {
|
||||
client.err = &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonProviderFailed, Message: "failed"}
|
||||
}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat bird"}}}, Validators: []contracts.Validator{check, rejectUntilRepair(0)}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat bird"}}}, Validators: []validators.Validator{check, rejectUntilRepair(0)}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{}}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -279,7 +290,7 @@ func TestRepairRevalidatesEarlierApprovalAndRetainsUnrelatedApprovalOnFailure(t
|
||||
func TestRepairInterruptionRetainsAttempt(t *testing.T) {
|
||||
client := &repairClient{t: t, err: &structuredoutput.Failure{Class: structuredoutput.FailureCanceled, Reason: outcomes.ReasonCanceled, Message: "canceled", Cause: context.Canceled}}
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -297,7 +308,7 @@ func TestRepairArtifactsAreUnique(t *testing.T) {
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "fox"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}},
|
||||
}
|
||||
section := 4
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar_1", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: candidates, SectionIndex: §ion, Exchange: &structuredoutput.Exchange{}, Interactions: llm.NewDiagnosticsWriter(t.TempDir())}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar_1", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{rejectUntilRepair()}, SectionEnriched: candidates, SectionIndex: §ion, Exchange: &structuredoutput.Exchange{}, Interactions: llm.NewDiagnosticsWriter(t.TempDir())}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -355,7 +366,7 @@ func TestRepairArtifactFailureDoesNotChangeOutcome(t *testing.T) {
|
||||
}
|
||||
sink := llm.NewDiagnosticsWriter(path)
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}, Interactions: sink}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}, Interactions: sink}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -382,8 +393,8 @@ func TestNoEffectAndUnavailableOnlyDoNotStartRepair(t *testing.T) {
|
||||
}
|
||||
|
||||
// rejectUntilRepair supplies a semantic rejection at the external validator seam.
|
||||
func rejectUntilRepair(preserved ...int) contracts.Validator {
|
||||
return fakeValidator{name: "editorial_review", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
func rejectUntilRepair(preserved ...int) validators.Validator {
|
||||
return fakeValidator{name: "editorial_review", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
var decisions []validators.Decision
|
||||
for _, p := range req.CandidateProposal {
|
||||
approved := req.RepairAttempt != nil
|
||||
|
||||
@@ -69,13 +69,14 @@ type WorkUnitOutcome struct {
|
||||
}
|
||||
|
||||
type ValidatorDecisionRecord struct {
|
||||
ValidatorName string `json:"validator_name"`
|
||||
ProposalIndex int `json:"proposal_index"`
|
||||
Approved bool `json:"approved"`
|
||||
Disposition validators.DecisionDisposition `json:"disposition"`
|
||||
ReasonCode string `json:"reason_code"`
|
||||
Message string `json:"message"`
|
||||
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
|
||||
ExecutionClass validatormetadata.ExecutionClass `json:"-"`
|
||||
ValidatorName string `json:"validator_name"`
|
||||
ProposalIndex int `json:"proposal_index"`
|
||||
Approved bool `json:"approved"`
|
||||
Disposition validators.DecisionDisposition `json:"disposition"`
|
||||
ReasonCode string `json:"reason_code"`
|
||||
Message string `json:"message"`
|
||||
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
|
||||
}
|
||||
|
||||
type ValidatorRejectedChange struct {
|
||||
@@ -650,7 +651,7 @@ type validateSectionCandidatesInput struct {
|
||||
Policy proposals.ReplacementPolicy
|
||||
WorkingTranscript *schema.Transcript
|
||||
SectionIndex *int
|
||||
Validators []contracts.Validator
|
||||
Validators []validators.Validator
|
||||
SectionEnriched []proposals.EnrichedCorrectionProposal
|
||||
ValidationLLMClient contracts.StructuredLLMClient
|
||||
Interactions structuredoutput.InteractionSink
|
||||
@@ -673,7 +674,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
||||
eligible := append([]proposals.EnrichedCorrectionProposal(nil), input.SectionEnriched...)
|
||||
feedback := input.Spec.Settings.LLM.CorrectionRepairAttempts > 0
|
||||
var blocked, noEffect map[int]bool
|
||||
evidence := contracts.ValidationRequest{WorkingTranscript: input.WorkingTranscript, Section: input.Section, ReplacementPolicy: input.Policy, IndependentEvidence: feedback}
|
||||
evidence := validators.Request{WorkingTranscript: input.WorkingTranscript, Section: input.Section, ReplacementPolicy: input.Policy, IndependentEvidence: feedback}
|
||||
if feedback {
|
||||
blocked, noEffect = make(map[int]bool), make(map[int]bool)
|
||||
evidence.Previews = make(map[int]proposals.SegmentPreviewResult)
|
||||
@@ -686,11 +687,12 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
||||
if err := ctx.Err(); err != nil {
|
||||
return validateSectionCandidatesResult{decisions: decisions, rejected: rejected, operations: operations, filtered: filtered}, err
|
||||
}
|
||||
class := validatormetadata.ClassOf(validator)
|
||||
submitted := eligible
|
||||
if feedback {
|
||||
submitted = feedbackCandidates(input, validator, noEffect, evidence)
|
||||
submitted = feedbackCandidates(input, validator.Name(), class == validatormetadata.ExecutionClassLLMBacked, noEffect, evidence)
|
||||
}
|
||||
vResult, err := validator.Validate(ctx, contracts.ValidationRequest{
|
||||
vResult, err := validator.Validate(ctx, validators.Request{
|
||||
StageIndex: &input.Spec.Index, StageID: input.Spec.ID,
|
||||
WorkingTranscript: input.WorkingTranscript,
|
||||
Section: evidence.Section, Previews: evidence.Previews,
|
||||
@@ -728,17 +730,20 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
||||
byIndex[p.ProposalIndex] = p
|
||||
}
|
||||
|
||||
for _, d := range vResult.Decisions {
|
||||
if feedback && validator.Name() == "original_text_presence" && d.Approved {
|
||||
preview := evidence.Preview(byIndex[d.ProposalIndex])
|
||||
if preview.SkipReason == proposals.SkipReasonAmbiguousOriginal {
|
||||
d.Approved = false
|
||||
d.Disposition = validators.DispositionRejected
|
||||
d.ReasonCode = string(proposals.SkipReasonAmbiguousOriginal)
|
||||
d.Message = "original_text matches multiple locations in the target segment"
|
||||
}
|
||||
dispositions := make([]validators.DecisionDisposition, len(vResult.Decisions))
|
||||
for i, decision := range vResult.Decisions {
|
||||
disposition, contractErr := decision.EffectiveDisposition()
|
||||
if contractErr != nil {
|
||||
return validateSectionCandidatesResult{
|
||||
decisions: decisions, rejected: rejected, operations: operations, filtered: filtered,
|
||||
}, fmt.Errorf("validator %q returned inconsistent decisions: %w", validator.Name(), errors.Join(err, contractErr))
|
||||
}
|
||||
if feedback && !d.Approved {
|
||||
dispositions[i] = disposition
|
||||
}
|
||||
for i, d := range vResult.Decisions {
|
||||
disposition := dispositions[i]
|
||||
approved := disposition == validators.DispositionApproved
|
||||
if feedback && !approved {
|
||||
blocked[d.ProposalIndex] = true
|
||||
if d.ReasonCode == validators.ReasonNoEffect {
|
||||
noEffect[d.ProposalIndex] = true
|
||||
@@ -746,18 +751,19 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
||||
}
|
||||
decisions = append(decisions, ValidatorDecisionRecord{
|
||||
ValidatorName: validator.Name(),
|
||||
ExecutionClass: class,
|
||||
ProposalIndex: d.ProposalIndex,
|
||||
Approved: d.Approved,
|
||||
Disposition: d.Disposition,
|
||||
Approved: approved,
|
||||
Disposition: disposition,
|
||||
ReasonCode: d.ReasonCode,
|
||||
Message: d.Message,
|
||||
DiagnosticArtifactPath: d.DiagnosticArtifactPath,
|
||||
})
|
||||
if d.Approved {
|
||||
if approved {
|
||||
nextEligible = append(nextEligible, byIndex[d.ProposalIndex])
|
||||
continue
|
||||
}
|
||||
if d.Disposition != validators.DispositionUnavailable {
|
||||
if disposition != validators.DispositionUnavailable {
|
||||
if reason, ok := validators.DefectiveProposalReason(d.ReasonCode); ok {
|
||||
filtered = append(filtered, outcomes.FilteredProposal{ProposalIndex: d.ProposalIndex, Reason: reason, Count: 1})
|
||||
}
|
||||
@@ -771,7 +777,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
||||
TargetSegmentID: p.TargetSegmentID,
|
||||
OriginalText: p.OriginalText,
|
||||
CorrectedText: p.CorrectedText,
|
||||
Disposition: d.Disposition,
|
||||
Disposition: disposition,
|
||||
ReasonCode: d.ReasonCode,
|
||||
Message: d.Message,
|
||||
})
|
||||
@@ -882,11 +888,13 @@ func attributeApplyFiltering(sections []chunking.Section, approved []proposals.E
|
||||
}
|
||||
}
|
||||
|
||||
func reorderValidatorsForPipeline(in []contracts.Validator) ([]contracts.Validator, map[string]int) {
|
||||
deterministic := make([]contracts.Validator, 0, len(in))
|
||||
llmBacked := make([]contracts.Validator, 0, len(in))
|
||||
func reorderValidatorsForPipeline(in []validators.Validator) ([]validators.Validator, map[string]int) {
|
||||
deterministic := make([]validators.Validator, 0, len(in))
|
||||
llmBacked := make([]validators.Validator, 0, len(in))
|
||||
for _, validator := range in {
|
||||
if validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked {
|
||||
class := validatormetadata.ClassOf(validator)
|
||||
validator = validatormetadata.Wrap(validator, class)
|
||||
if class == validatormetadata.ExecutionClassLLMBacked {
|
||||
llmBacked = append(llmBacked, validator)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -43,7 +43,7 @@ func (f fakeFactory) ModuleForSpec(spec contracts.ModuleRunSpec) (contracts.Tran
|
||||
type fakeModule struct {
|
||||
key string
|
||||
policy proposals.ReplacementPolicy
|
||||
validators []contracts.Validator
|
||||
validators []validators.Validator
|
||||
proposeF func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error)
|
||||
}
|
||||
|
||||
@@ -52,6 +52,33 @@ type outcomeModule struct {
|
||||
proposeResult func(contracts.ProposalRequest) (contracts.ProposalResult, error)
|
||||
}
|
||||
|
||||
func TestRunnerStopsOnContradictoryValidatorDecision(t *testing.T) {
|
||||
candidate := proposals.EnrichedCorrectionProposal{
|
||||
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"},
|
||||
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 7},
|
||||
}
|
||||
contractFailure := errors.New("validator also returned a runtime error")
|
||||
for _, cause := range []error{nil, contractFailure} {
|
||||
contradictory := fakeValidator{name: "invalid", validateF: func(validators.Request) (validators.Result, error) {
|
||||
return validators.Result{Decisions: []validators.Decision{{
|
||||
ProposalIndex: 7, Approved: true, Disposition: validators.DispositionUnavailable,
|
||||
}}}, cause
|
||||
}}
|
||||
later := fakeValidator{name: "later", validateF: func(validators.Request) (validators.Result, error) {
|
||||
t.Fatal("candidate advanced past a contradictory decision")
|
||||
return validators.Result{}, nil
|
||||
}}
|
||||
result, err := validateSectionCandidates(context.Background(), validateSectionCandidatesInput{
|
||||
Validators: []validators.Validator{validators.ProposalShapeValidator{}, contradictory, later},
|
||||
SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate},
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "contradictory") || (cause != nil && !errors.Is(err, cause)) ||
|
||||
len(result.approved) != 0 || len(result.decisions) != 1 || result.decisions[0].ValidatorName != "proposal_shape" {
|
||||
t.Fatalf("contradictory result = %+v (%v)", result, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRecoverableProposalFailureAllowsLaterModule(t *testing.T) {
|
||||
failed := outcomeModule{fakeModule: fakeModule{key: "first", policy: proposals.ReplacementPolicyRequireUnique}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
return contracts.ProposalResult{Operations: []outcomes.Operation{{Reference: outcomes.Reference{ModuleKey: "first", ModuleInstance: "first", Kind: outcomes.OperationProposal}, FailureReason: outcomes.ReasonTransportFailed}}}, nil
|
||||
@@ -74,7 +101,7 @@ func TestRunnerRecoverableProposalFailureAllowsLaterModule(t *testing.T) {
|
||||
|
||||
func TestRunnerOnlyFullyValidatedCandidatesApplyAndRecoveriesMerge(t *testing.T) {
|
||||
secondSaw := make([]int, 0)
|
||||
partial := fakeValidator{name: "partial", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
partial := fakeValidator{name: "partial", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
if len(req.CandidateProposal) != 2 {
|
||||
t.Fatalf("expected two candidates after shape filtering, got %d", len(req.CandidateProposal))
|
||||
}
|
||||
@@ -86,13 +113,13 @@ func TestRunnerOnlyFullyValidatedCandidatesApplyAndRecoveriesMerge(t *testing.T)
|
||||
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation}, FailureReason: outcomes.ReasonProviderFailed},
|
||||
}}, nil
|
||||
}}
|
||||
last := fakeValidator{name: "last", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
last := fakeValidator{name: "last", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
for _, candidate := range req.CandidateProposal {
|
||||
secondSaw = append(secondSaw, candidate.ProposalIndex)
|
||||
}
|
||||
return validators.Result{ValidatorName: "last", Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: true, Disposition: validators.DispositionApproved}}}, nil
|
||||
}}
|
||||
module := outcomeModule{fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{validators.ProposalShapeValidator{}, partial, last}}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
module := outcomeModule{fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{validators.ProposalShapeValidator{}, partial, last}}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
return contracts.ProposalResult{Proposals: []proposals.CorrectionProposal{
|
||||
{TargetSegmentID: 0, OriginalText: "bad", CorrectedText: "x"},
|
||||
{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"},
|
||||
@@ -120,10 +147,10 @@ func TestRunnerOnlyFullyValidatedCandidatesApplyAndRecoveriesMerge(t *testing.T)
|
||||
func TestFatalValidatorNeverReturnsUncheckedCandidates(t *testing.T) {
|
||||
origin := errors.New("validator setup failed")
|
||||
proposal := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 7}}
|
||||
broken := fakeValidator{name: "broken", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
broken := fakeValidator{name: "broken", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
return validators.Result{Operations: []outcomes.Operation{{Reference: outcomes.Reference{Kind: outcomes.OperationValidation}, FailureReason: outcomes.ReasonSetupFailure}}}, origin
|
||||
}}
|
||||
result, err := validateSectionCandidates(context.Background(), validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "m", InstanceName: "m"}, Validators: []contracts.Validator{broken}, SectionEnriched: []proposals.EnrichedCorrectionProposal{proposal}})
|
||||
result, err := validateSectionCandidates(context.Background(), validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "m", InstanceName: "m"}, Validators: []validators.Validator{broken}, SectionEnriched: []proposals.EnrichedCorrectionProposal{proposal}})
|
||||
if !errors.Is(err, origin) || len(result.approved) != 0 || len(result.operations) != 1 {
|
||||
t.Fatalf("fatal result must retain facts without approving candidates: result=%+v err=%v", result, err)
|
||||
}
|
||||
@@ -131,7 +158,7 @@ func TestFatalValidatorNeverReturnsUncheckedCandidates(t *testing.T) {
|
||||
|
||||
func TestCandidateCaptureIsExplicitAndIncludesFilteredProposals(t *testing.T) {
|
||||
module := fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique,
|
||||
validators: []contracts.Validator{validators.ProposalShapeValidator{}},
|
||||
validators: []validators.Validator{validators.ProposalShapeValidator{}},
|
||||
proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 0, OriginalText: "bad", CorrectedText: "good"}}, nil
|
||||
}}
|
||||
@@ -160,7 +187,7 @@ func (m outcomeModule) Propose(_ context.Context, req contracts.ProposalRequest)
|
||||
|
||||
func (m fakeModule) Key() string { return m.key }
|
||||
func (m fakeModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy }
|
||||
func (m fakeModule) Validators() []contracts.Validator { return m.validators }
|
||||
func (m fakeModule) Validators() []validators.Validator { return m.validators }
|
||||
func (m fakeModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
if m.proposeF == nil {
|
||||
return contracts.ProposalResult{}, nil
|
||||
@@ -171,11 +198,11 @@ func (m fakeModule) Propose(ctx context.Context, req contracts.ProposalRequest)
|
||||
|
||||
type fakeValidator struct {
|
||||
name string
|
||||
validateF func(req contracts.ValidationRequest) (validators.Result, error)
|
||||
validateF func(req validators.Request) (validators.Result, error)
|
||||
}
|
||||
|
||||
func (v fakeValidator) Name() string { return v.name }
|
||||
func (v fakeValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (validators.Result, error) {
|
||||
func (v fakeValidator) Validate(ctx context.Context, req validators.Request) (validators.Result, error) {
|
||||
_ = ctx
|
||||
return v.validateF(req)
|
||||
}
|
||||
@@ -191,19 +218,19 @@ func (v classifiedFakeValidator) ExecutionClass() validatormetadata.ExecutionCla
|
||||
|
||||
func TestReorderValidatorsDeterministicBeforeLLMBacked(t *testing.T) {
|
||||
llm := classifiedFakeValidator{
|
||||
fakeValidator: fakeValidator{name: "llm", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
fakeValidator: fakeValidator{name: "llm", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
return validators.Result{ValidatorName: "llm"}, nil
|
||||
}},
|
||||
class: validatormetadata.ExecutionClassLLMBacked,
|
||||
}
|
||||
deterministic := classifiedFakeValidator{
|
||||
fakeValidator: fakeValidator{name: "deterministic", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
fakeValidator: fakeValidator{name: "deterministic", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
return validators.Result{ValidatorName: "deterministic"}, nil
|
||||
}},
|
||||
class: validatormetadata.ExecutionClassDeterministic,
|
||||
}
|
||||
|
||||
ordered, _ := reorderValidatorsForPipeline([]contracts.Validator{llm, deterministic})
|
||||
ordered, _ := reorderValidatorsForPipeline([]validators.Validator{llm, deterministic})
|
||||
if len(ordered) != 2 {
|
||||
t.Fatalf("expected 2 validators, got %d", len(ordered))
|
||||
}
|
||||
@@ -213,17 +240,17 @@ func TestReorderValidatorsDeterministicBeforeLLMBacked(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestReorderValidatorsDefaultsUnclassifiedToDeterministic(t *testing.T) {
|
||||
unclassified := fakeValidator{name: "plain", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
unclassified := fakeValidator{name: "plain", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
return validators.Result{ValidatorName: "plain"}, nil
|
||||
}}
|
||||
llm := classifiedFakeValidator{
|
||||
fakeValidator: fakeValidator{name: "llm", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
fakeValidator: fakeValidator{name: "llm", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
return validators.Result{ValidatorName: "llm"}, nil
|
||||
}},
|
||||
class: validatormetadata.ExecutionClassLLMBacked,
|
||||
}
|
||||
|
||||
ordered, _ := reorderValidatorsForPipeline([]contracts.Validator{llm, unclassified})
|
||||
ordered, _ := reorderValidatorsForPipeline([]validators.Validator{llm, unclassified})
|
||||
if len(ordered) != 2 {
|
||||
t.Fatalf("expected 2 validators, got %d", len(ordered))
|
||||
}
|
||||
@@ -553,7 +580,7 @@ func TestRunnerValidationStartsBeforeAllSectionProposalsComplete(t *testing.T) {
|
||||
|
||||
validator := fakeValidator{
|
||||
name: "capture",
|
||||
validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
validateF: func(req validators.Request) (validators.Result, error) {
|
||||
if len(req.CandidateProposal) == 1 && req.CandidateProposal[0].TargetSegmentID == 1 {
|
||||
select {
|
||||
case validatorStarted <- struct{}{}:
|
||||
@@ -577,7 +604,7 @@ func TestRunnerValidationStartsBeforeAllSectionProposalsComplete(t *testing.T) {
|
||||
"m": fakeModule{
|
||||
key: "m",
|
||||
policy: proposals.ReplacementPolicyRequireUnique,
|
||||
validators: []contracts.Validator{validator},
|
||||
validators: []validators.Validator{validator},
|
||||
proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
seg := req.WorkingTranscript.Segments[0]
|
||||
if req.Section != nil && req.Section.Index == 1 {
|
||||
@@ -630,7 +657,7 @@ func (c *captureContractStructuredClient) CompleteStructured(ctx context.Context
|
||||
if target, ok := out.(*validators.LLMValidationResponse); ok {
|
||||
*target = validators.LLMValidationResponse{
|
||||
Validations: []validators.LLMValidationDecision{
|
||||
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
|
||||
{CorrectionIndex: "1", Approved: true, Reason: "ok"},
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -662,7 +689,7 @@ func (c *stageAwareStructuredClient) CompleteStructured(ctx context.Context, req
|
||||
target := out.(*validators.LLMValidationResponse)
|
||||
*target = validators.LLMValidationResponse{
|
||||
Validations: []validators.LLMValidationDecision{
|
||||
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
|
||||
{CorrectionIndex: "1", Approved: true, Reason: "ok"},
|
||||
},
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{}, nil
|
||||
@@ -734,7 +761,7 @@ func TestRunnerRepeatedModuleInstanceNames(t *testing.T) {
|
||||
|
||||
func TestRunnerValidatorApprovedProposalApplied(t *testing.T) {
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}}
|
||||
allowAll := fakeValidator{name: "allow", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
allowAll := fakeValidator{name: "allow", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
decisions := make([]validators.Decision, len(req.CandidateProposal))
|
||||
for i, p := range req.CandidateProposal {
|
||||
decisions[i] = validators.Decision{ProposalIndex: p.ProposalIndex, Approved: true, ReasonCode: validators.ReasonApproved, Message: "approved"}
|
||||
@@ -745,7 +772,7 @@ func TestRunnerValidatorApprovedProposalApplied(t *testing.T) {
|
||||
"m": fakeModule{
|
||||
key: "m",
|
||||
policy: proposals.ReplacementPolicyRequireUnique,
|
||||
validators: []contracts.Validator{allowAll},
|
||||
validators: []validators.Validator{allowAll},
|
||||
proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil
|
||||
},
|
||||
@@ -766,7 +793,7 @@ func TestRunnerValidatorApprovedProposalApplied(t *testing.T) {
|
||||
|
||||
func TestRunnerValidatorRejectedProposalNotApplied(t *testing.T) {
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}}
|
||||
rejectAll := fakeValidator{name: "reject", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
rejectAll := fakeValidator{name: "reject", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
decisions := make([]validators.Decision, len(req.CandidateProposal))
|
||||
for i, p := range req.CandidateProposal {
|
||||
decisions[i] = validators.Decision{ProposalIndex: p.ProposalIndex, Approved: false, ReasonCode: validators.ReasonNoEffect, Message: "rejected"}
|
||||
@@ -774,7 +801,7 @@ func TestRunnerValidatorRejectedProposalNotApplied(t *testing.T) {
|
||||
return validators.Result{ValidatorName: "reject", Decisions: decisions}, nil
|
||||
}}
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{rejectAll}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{rejectAll}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil
|
||||
}},
|
||||
}})
|
||||
@@ -792,7 +819,7 @@ func TestRunnerValidatorRejectedProposalNotApplied(t *testing.T) {
|
||||
|
||||
func TestRunnerMultipleValidatorsRunInOrderAndFilterSurvivors(t *testing.T) {
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "one two"}}}
|
||||
first := fakeValidator{name: "first", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
first := fakeValidator{name: "first", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
if len(req.CandidateProposal) != 2 {
|
||||
t.Fatalf("expected first validator to see 2 candidates, got %d", len(req.CandidateProposal))
|
||||
}
|
||||
@@ -804,14 +831,14 @@ func TestRunnerMultipleValidatorsRunInOrderAndFilterSurvivors(t *testing.T) {
|
||||
},
|
||||
}, nil
|
||||
}}
|
||||
second := fakeValidator{name: "second", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
second := fakeValidator{name: "second", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
if len(req.CandidateProposal) != 1 {
|
||||
t.Fatalf("expected second validator to see only survivors, got %d", len(req.CandidateProposal))
|
||||
}
|
||||
return validators.Result{ValidatorName: "second", Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: true, ReasonCode: validators.ReasonApproved, Message: "ok"}}}, nil
|
||||
}}
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{first, second}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{first, second}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{
|
||||
{TargetSegmentID: 1, OriginalText: "one", CorrectedText: "ONE"},
|
||||
{TargetSegmentID: 1, OriginalText: "two", CorrectedText: "TWO"},
|
||||
@@ -833,11 +860,11 @@ func TestRunnerValidatorCardinalityErrorStopsPipelineWithPartialProgress(t *test
|
||||
good := fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil
|
||||
}}
|
||||
badValidator := fakeValidator{name: "bad", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
badValidator := fakeValidator{name: "bad", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
// Missing one decision triggers cardinality error.
|
||||
return validators.Result{ValidatorName: "bad", Decisions: nil}, nil
|
||||
}}
|
||||
bad := fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{badValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
bad := fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{badValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}}, nil
|
||||
}}
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"m1": good, "m2": bad}})
|
||||
@@ -856,11 +883,11 @@ func TestRunnerValidatorCardinalityErrorStopsPipelineWithPartialProgress(t *test
|
||||
|
||||
func TestRunnerApplicationSkipAfterValidatorApprovalReported(t *testing.T) {
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "word word"}}}
|
||||
allow := fakeValidator{name: "allow", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
allow := fakeValidator{name: "allow", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
return validators.Result{ValidatorName: "allow", Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: true, ReasonCode: validators.ReasonApproved, Message: "ok"}}}, nil
|
||||
}}
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{allow}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{allow}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "word", CorrectedText: "term"}}, nil
|
||||
}},
|
||||
}})
|
||||
@@ -880,7 +907,7 @@ func TestRunnerApplyConflictAfterTwoInitiallyValidDeletions(t *testing.T) {
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "a b"}}}
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": outcomeModule{
|
||||
fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NonEmptyResultingSegmentValidator{}}},
|
||||
fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NonEmptyResultingSegmentValidator{}}},
|
||||
proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
return contracts.ProposalResult{
|
||||
Proposals: []proposals.CorrectionProposal{
|
||||
@@ -945,14 +972,14 @@ func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contr
|
||||
}
|
||||
|
||||
func TestRunnerLLMValidatorApprovalApplied(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}}}
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}}}
|
||||
llmValidator, err := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
|
||||
if err != nil {
|
||||
t.Fatalf("NewLLMBackedValidator: %v", err)
|
||||
}
|
||||
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil
|
||||
}},
|
||||
}})
|
||||
@@ -971,10 +998,10 @@ func TestRunnerLLMValidatorApprovalApplied(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunnerLLMValidatorRejectionPreventsApplication(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: false, Reason: "reject"}}}}}
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: false, Reason: "reject"}}}}}
|
||||
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil
|
||||
}},
|
||||
}})
|
||||
@@ -996,13 +1023,13 @@ func TestRunnerLLMValidatorRejectionPreventsApplication(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Reason: "bad index"}}}}}
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "99", Approved: true, Reason: "bad index"}}}}}
|
||||
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m1": fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil
|
||||
}},
|
||||
"m2": fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
"m2": fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}}, nil
|
||||
}},
|
||||
}})
|
||||
@@ -1028,13 +1055,13 @@ func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress(
|
||||
|
||||
func TestRunnerReconciledSurvivorsCompleteLaterValidation(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{
|
||||
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
|
||||
{CorrectionIndex: 3, Approved: true, Reason: "ok"},
|
||||
{CorrectionIndex: 99, Approved: true, Reason: "unexpected"},
|
||||
{CorrectionIndex: "1", Approved: true, Reason: "ok"},
|
||||
{CorrectionIndex: "3", Approved: true, Reason: "ok"},
|
||||
{CorrectionIndex: "99", Approved: true, Reason: "unexpected"},
|
||||
}}}}
|
||||
first, _ := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
|
||||
var laterIndices []int
|
||||
later := fakeValidator{name: "later_review", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
later := fakeValidator{name: "later_review", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
decisions := make([]validators.Decision, 0, len(req.CandidateProposal))
|
||||
for _, candidate := range req.CandidateProposal {
|
||||
laterIndices = append(laterIndices, candidate.ProposalIndex)
|
||||
@@ -1047,7 +1074,7 @@ func TestRunnerReconciledSurvivorsCompleteLaterValidation(t *testing.T) {
|
||||
}
|
||||
return validators.Result{ValidatorName: "later_review", Decisions: decisions}, nil
|
||||
}}
|
||||
module := outcomeModule{fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{first, validatormetadata.Wrap(later, validatormetadata.ExecutionClassLLMBacked)}}, proposeResult: func(contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
module := outcomeModule{fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{first, validatormetadata.Wrap(later, validatormetadata.ExecutionClassLLMBacked)}}, proposeResult: func(contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
return contracts.ProposalResult{Proposals: []proposals.CorrectionProposal{
|
||||
{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"},
|
||||
{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"},
|
||||
@@ -1077,7 +1104,7 @@ func TestRunnerLLMValidatorMissingDecisionRejectsBatch(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{}}}}
|
||||
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil
|
||||
}},
|
||||
}})
|
||||
@@ -1094,12 +1121,12 @@ func TestRunnerLLMValidatorMissingDecisionRejectsBatch(t *testing.T) {
|
||||
|
||||
func TestRunnerLLMValidatorDuplicateDecisionRejectsBatch(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{
|
||||
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
|
||||
{CorrectionIndex: 1, Approved: false, Reason: "dup"},
|
||||
{CorrectionIndex: "1", Approved: true, Reason: "ok"},
|
||||
{CorrectionIndex: "1", Approved: false, Reason: "dup"},
|
||||
}}}}
|
||||
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil
|
||||
}},
|
||||
}})
|
||||
@@ -1120,8 +1147,8 @@ func (lenEstimator) EstimateTokens(text string) int { return len(text) }
|
||||
|
||||
func TestRunnerLLMValidatorBatching(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{
|
||||
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}},
|
||||
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}},
|
||||
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}},
|
||||
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}},
|
||||
}}
|
||||
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
|
||||
llmValidator.SetTokenEstimator(lenEstimator{})
|
||||
@@ -1129,7 +1156,7 @@ func TestRunnerLLMValidatorBatching(t *testing.T) {
|
||||
cfg := runnerTestConfig()
|
||||
cfg.Validation.MaxPromptTokens = 260
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyReplaceAll, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyReplaceAll, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{
|
||||
{TargetSegmentID: 1, OriginalText: "alpha", CorrectedText: strings.Repeat("B", 40)},
|
||||
{TargetSegmentID: 1, OriginalText: "gamma", CorrectedText: strings.Repeat("D", 40)},
|
||||
@@ -1154,12 +1181,12 @@ func TestRunnerLLMValidatorBatching(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunnerLLMValidatorDiagnosticsWritten(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "supported by context"}}}}}
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "supported by context"}}}}}
|
||||
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
|
||||
cfg := runnerTestConfig()
|
||||
diagDir := t.TempDir()
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil
|
||||
}},
|
||||
}})
|
||||
@@ -1208,7 +1235,7 @@ type proposalGenerationModule struct {
|
||||
|
||||
func (m proposalGenerationModule) Key() string { return m.key }
|
||||
func (m proposalGenerationModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy }
|
||||
func (m proposalGenerationModule) Validators() []contracts.Validator { return nil }
|
||||
func (m proposalGenerationModule) Validators() []validators.Validator { return nil }
|
||||
func (m proposalGenerationModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
result, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
|
||||
ModuleKey: req.RunSpec.ModuleKey,
|
||||
@@ -1294,10 +1321,10 @@ func TestFatalPartialDecisionsRejectUnknownAndDuplicateIdentity(t *testing.T) {
|
||||
{{ProposalIndex: 8, Approved: true}},
|
||||
{{ProposalIndex: 7, Approved: true}, {ProposalIndex: 7, Approved: true}},
|
||||
} {
|
||||
v := fakeValidator{name: "broken", validateF: func(contracts.ValidationRequest) (validators.Result, error) {
|
||||
v := fakeValidator{name: "broken", validateF: func(validators.Request) (validators.Result, error) {
|
||||
return validators.Result{Decisions: decisions}, origin
|
||||
}}
|
||||
result, err := validateSectionCandidates(context.Background(), validateSectionCandidatesInput{Validators: []contracts.Validator{v}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}})
|
||||
result, err := validateSectionCandidates(context.Background(), validateSectionCandidatesInput{Validators: []validators.Validator{v}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}})
|
||||
if !errors.Is(err, origin) || len(result.decisions) != 0 || len(result.approved) != 0 {
|
||||
t.Fatalf("invalid partial evidence accepted or origin lost: %+v %v", result, err)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user