Validate internal decisions and reconcile exact model references
This commit is contained in:
@@ -43,7 +43,7 @@ func (f fakeFactory) ModuleForSpec(spec contracts.ModuleRunSpec) (contracts.Tran
|
||||
type fakeModule struct {
|
||||
key string
|
||||
policy proposals.ReplacementPolicy
|
||||
validators []contracts.Validator
|
||||
validators []validators.Validator
|
||||
proposeF func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error)
|
||||
}
|
||||
|
||||
@@ -52,6 +52,33 @@ type outcomeModule struct {
|
||||
proposeResult func(contracts.ProposalRequest) (contracts.ProposalResult, error)
|
||||
}
|
||||
|
||||
func TestRunnerStopsOnContradictoryValidatorDecision(t *testing.T) {
|
||||
candidate := proposals.EnrichedCorrectionProposal{
|
||||
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"},
|
||||
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 7},
|
||||
}
|
||||
contractFailure := errors.New("validator also returned a runtime error")
|
||||
for _, cause := range []error{nil, contractFailure} {
|
||||
contradictory := fakeValidator{name: "invalid", validateF: func(validators.Request) (validators.Result, error) {
|
||||
return validators.Result{Decisions: []validators.Decision{{
|
||||
ProposalIndex: 7, Approved: true, Disposition: validators.DispositionUnavailable,
|
||||
}}}, cause
|
||||
}}
|
||||
later := fakeValidator{name: "later", validateF: func(validators.Request) (validators.Result, error) {
|
||||
t.Fatal("candidate advanced past a contradictory decision")
|
||||
return validators.Result{}, nil
|
||||
}}
|
||||
result, err := validateSectionCandidates(context.Background(), validateSectionCandidatesInput{
|
||||
Validators: []validators.Validator{validators.ProposalShapeValidator{}, contradictory, later},
|
||||
SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate},
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "contradictory") || (cause != nil && !errors.Is(err, cause)) ||
|
||||
len(result.approved) != 0 || len(result.decisions) != 1 || result.decisions[0].ValidatorName != "proposal_shape" {
|
||||
t.Fatalf("contradictory result = %+v (%v)", result, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRecoverableProposalFailureAllowsLaterModule(t *testing.T) {
|
||||
failed := outcomeModule{fakeModule: fakeModule{key: "first", policy: proposals.ReplacementPolicyRequireUnique}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
return contracts.ProposalResult{Operations: []outcomes.Operation{{Reference: outcomes.Reference{ModuleKey: "first", ModuleInstance: "first", Kind: outcomes.OperationProposal}, FailureReason: outcomes.ReasonTransportFailed}}}, nil
|
||||
@@ -74,7 +101,7 @@ func TestRunnerRecoverableProposalFailureAllowsLaterModule(t *testing.T) {
|
||||
|
||||
func TestRunnerOnlyFullyValidatedCandidatesApplyAndRecoveriesMerge(t *testing.T) {
|
||||
secondSaw := make([]int, 0)
|
||||
partial := fakeValidator{name: "partial", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
partial := fakeValidator{name: "partial", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
if len(req.CandidateProposal) != 2 {
|
||||
t.Fatalf("expected two candidates after shape filtering, got %d", len(req.CandidateProposal))
|
||||
}
|
||||
@@ -86,13 +113,13 @@ func TestRunnerOnlyFullyValidatedCandidatesApplyAndRecoveriesMerge(t *testing.T)
|
||||
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation}, FailureReason: outcomes.ReasonProviderFailed},
|
||||
}}, nil
|
||||
}}
|
||||
last := fakeValidator{name: "last", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
last := fakeValidator{name: "last", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
for _, candidate := range req.CandidateProposal {
|
||||
secondSaw = append(secondSaw, candidate.ProposalIndex)
|
||||
}
|
||||
return validators.Result{ValidatorName: "last", Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: true, Disposition: validators.DispositionApproved}}}, nil
|
||||
}}
|
||||
module := outcomeModule{fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{validators.ProposalShapeValidator{}, partial, last}}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
module := outcomeModule{fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{validators.ProposalShapeValidator{}, partial, last}}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
return contracts.ProposalResult{Proposals: []proposals.CorrectionProposal{
|
||||
{TargetSegmentID: 0, OriginalText: "bad", CorrectedText: "x"},
|
||||
{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"},
|
||||
@@ -120,10 +147,10 @@ func TestRunnerOnlyFullyValidatedCandidatesApplyAndRecoveriesMerge(t *testing.T)
|
||||
func TestFatalValidatorNeverReturnsUncheckedCandidates(t *testing.T) {
|
||||
origin := errors.New("validator setup failed")
|
||||
proposal := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 7}}
|
||||
broken := fakeValidator{name: "broken", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
broken := fakeValidator{name: "broken", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
return validators.Result{Operations: []outcomes.Operation{{Reference: outcomes.Reference{Kind: outcomes.OperationValidation}, FailureReason: outcomes.ReasonSetupFailure}}}, origin
|
||||
}}
|
||||
result, err := validateSectionCandidates(context.Background(), validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "m", InstanceName: "m"}, Validators: []contracts.Validator{broken}, SectionEnriched: []proposals.EnrichedCorrectionProposal{proposal}})
|
||||
result, err := validateSectionCandidates(context.Background(), validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "m", InstanceName: "m"}, Validators: []validators.Validator{broken}, SectionEnriched: []proposals.EnrichedCorrectionProposal{proposal}})
|
||||
if !errors.Is(err, origin) || len(result.approved) != 0 || len(result.operations) != 1 {
|
||||
t.Fatalf("fatal result must retain facts without approving candidates: result=%+v err=%v", result, err)
|
||||
}
|
||||
@@ -131,7 +158,7 @@ func TestFatalValidatorNeverReturnsUncheckedCandidates(t *testing.T) {
|
||||
|
||||
func TestCandidateCaptureIsExplicitAndIncludesFilteredProposals(t *testing.T) {
|
||||
module := fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique,
|
||||
validators: []contracts.Validator{validators.ProposalShapeValidator{}},
|
||||
validators: []validators.Validator{validators.ProposalShapeValidator{}},
|
||||
proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 0, OriginalText: "bad", CorrectedText: "good"}}, nil
|
||||
}}
|
||||
@@ -160,7 +187,7 @@ func (m outcomeModule) Propose(_ context.Context, req contracts.ProposalRequest)
|
||||
|
||||
func (m fakeModule) Key() string { return m.key }
|
||||
func (m fakeModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy }
|
||||
func (m fakeModule) Validators() []contracts.Validator { return m.validators }
|
||||
func (m fakeModule) Validators() []validators.Validator { return m.validators }
|
||||
func (m fakeModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
if m.proposeF == nil {
|
||||
return contracts.ProposalResult{}, nil
|
||||
@@ -171,11 +198,11 @@ func (m fakeModule) Propose(ctx context.Context, req contracts.ProposalRequest)
|
||||
|
||||
type fakeValidator struct {
|
||||
name string
|
||||
validateF func(req contracts.ValidationRequest) (validators.Result, error)
|
||||
validateF func(req validators.Request) (validators.Result, error)
|
||||
}
|
||||
|
||||
func (v fakeValidator) Name() string { return v.name }
|
||||
func (v fakeValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (validators.Result, error) {
|
||||
func (v fakeValidator) Validate(ctx context.Context, req validators.Request) (validators.Result, error) {
|
||||
_ = ctx
|
||||
return v.validateF(req)
|
||||
}
|
||||
@@ -191,19 +218,19 @@ func (v classifiedFakeValidator) ExecutionClass() validatormetadata.ExecutionCla
|
||||
|
||||
func TestReorderValidatorsDeterministicBeforeLLMBacked(t *testing.T) {
|
||||
llm := classifiedFakeValidator{
|
||||
fakeValidator: fakeValidator{name: "llm", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
fakeValidator: fakeValidator{name: "llm", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
return validators.Result{ValidatorName: "llm"}, nil
|
||||
}},
|
||||
class: validatormetadata.ExecutionClassLLMBacked,
|
||||
}
|
||||
deterministic := classifiedFakeValidator{
|
||||
fakeValidator: fakeValidator{name: "deterministic", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
fakeValidator: fakeValidator{name: "deterministic", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
return validators.Result{ValidatorName: "deterministic"}, nil
|
||||
}},
|
||||
class: validatormetadata.ExecutionClassDeterministic,
|
||||
}
|
||||
|
||||
ordered, _ := reorderValidatorsForPipeline([]contracts.Validator{llm, deterministic})
|
||||
ordered, _ := reorderValidatorsForPipeline([]validators.Validator{llm, deterministic})
|
||||
if len(ordered) != 2 {
|
||||
t.Fatalf("expected 2 validators, got %d", len(ordered))
|
||||
}
|
||||
@@ -213,17 +240,17 @@ func TestReorderValidatorsDeterministicBeforeLLMBacked(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestReorderValidatorsDefaultsUnclassifiedToDeterministic(t *testing.T) {
|
||||
unclassified := fakeValidator{name: "plain", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
unclassified := fakeValidator{name: "plain", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
return validators.Result{ValidatorName: "plain"}, nil
|
||||
}}
|
||||
llm := classifiedFakeValidator{
|
||||
fakeValidator: fakeValidator{name: "llm", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
fakeValidator: fakeValidator{name: "llm", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
return validators.Result{ValidatorName: "llm"}, nil
|
||||
}},
|
||||
class: validatormetadata.ExecutionClassLLMBacked,
|
||||
}
|
||||
|
||||
ordered, _ := reorderValidatorsForPipeline([]contracts.Validator{llm, unclassified})
|
||||
ordered, _ := reorderValidatorsForPipeline([]validators.Validator{llm, unclassified})
|
||||
if len(ordered) != 2 {
|
||||
t.Fatalf("expected 2 validators, got %d", len(ordered))
|
||||
}
|
||||
@@ -553,7 +580,7 @@ func TestRunnerValidationStartsBeforeAllSectionProposalsComplete(t *testing.T) {
|
||||
|
||||
validator := fakeValidator{
|
||||
name: "capture",
|
||||
validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
validateF: func(req validators.Request) (validators.Result, error) {
|
||||
if len(req.CandidateProposal) == 1 && req.CandidateProposal[0].TargetSegmentID == 1 {
|
||||
select {
|
||||
case validatorStarted <- struct{}{}:
|
||||
@@ -577,7 +604,7 @@ func TestRunnerValidationStartsBeforeAllSectionProposalsComplete(t *testing.T) {
|
||||
"m": fakeModule{
|
||||
key: "m",
|
||||
policy: proposals.ReplacementPolicyRequireUnique,
|
||||
validators: []contracts.Validator{validator},
|
||||
validators: []validators.Validator{validator},
|
||||
proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
seg := req.WorkingTranscript.Segments[0]
|
||||
if req.Section != nil && req.Section.Index == 1 {
|
||||
@@ -630,7 +657,7 @@ func (c *captureContractStructuredClient) CompleteStructured(ctx context.Context
|
||||
if target, ok := out.(*validators.LLMValidationResponse); ok {
|
||||
*target = validators.LLMValidationResponse{
|
||||
Validations: []validators.LLMValidationDecision{
|
||||
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
|
||||
{CorrectionIndex: "1", Approved: true, Reason: "ok"},
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -662,7 +689,7 @@ func (c *stageAwareStructuredClient) CompleteStructured(ctx context.Context, req
|
||||
target := out.(*validators.LLMValidationResponse)
|
||||
*target = validators.LLMValidationResponse{
|
||||
Validations: []validators.LLMValidationDecision{
|
||||
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
|
||||
{CorrectionIndex: "1", Approved: true, Reason: "ok"},
|
||||
},
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{}, nil
|
||||
@@ -734,7 +761,7 @@ func TestRunnerRepeatedModuleInstanceNames(t *testing.T) {
|
||||
|
||||
func TestRunnerValidatorApprovedProposalApplied(t *testing.T) {
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}}
|
||||
allowAll := fakeValidator{name: "allow", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
allowAll := fakeValidator{name: "allow", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
decisions := make([]validators.Decision, len(req.CandidateProposal))
|
||||
for i, p := range req.CandidateProposal {
|
||||
decisions[i] = validators.Decision{ProposalIndex: p.ProposalIndex, Approved: true, ReasonCode: validators.ReasonApproved, Message: "approved"}
|
||||
@@ -745,7 +772,7 @@ func TestRunnerValidatorApprovedProposalApplied(t *testing.T) {
|
||||
"m": fakeModule{
|
||||
key: "m",
|
||||
policy: proposals.ReplacementPolicyRequireUnique,
|
||||
validators: []contracts.Validator{allowAll},
|
||||
validators: []validators.Validator{allowAll},
|
||||
proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil
|
||||
},
|
||||
@@ -766,7 +793,7 @@ func TestRunnerValidatorApprovedProposalApplied(t *testing.T) {
|
||||
|
||||
func TestRunnerValidatorRejectedProposalNotApplied(t *testing.T) {
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}}
|
||||
rejectAll := fakeValidator{name: "reject", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
rejectAll := fakeValidator{name: "reject", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
decisions := make([]validators.Decision, len(req.CandidateProposal))
|
||||
for i, p := range req.CandidateProposal {
|
||||
decisions[i] = validators.Decision{ProposalIndex: p.ProposalIndex, Approved: false, ReasonCode: validators.ReasonNoEffect, Message: "rejected"}
|
||||
@@ -774,7 +801,7 @@ func TestRunnerValidatorRejectedProposalNotApplied(t *testing.T) {
|
||||
return validators.Result{ValidatorName: "reject", Decisions: decisions}, nil
|
||||
}}
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{rejectAll}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{rejectAll}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil
|
||||
}},
|
||||
}})
|
||||
@@ -792,7 +819,7 @@ func TestRunnerValidatorRejectedProposalNotApplied(t *testing.T) {
|
||||
|
||||
func TestRunnerMultipleValidatorsRunInOrderAndFilterSurvivors(t *testing.T) {
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "one two"}}}
|
||||
first := fakeValidator{name: "first", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
first := fakeValidator{name: "first", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
if len(req.CandidateProposal) != 2 {
|
||||
t.Fatalf("expected first validator to see 2 candidates, got %d", len(req.CandidateProposal))
|
||||
}
|
||||
@@ -804,14 +831,14 @@ func TestRunnerMultipleValidatorsRunInOrderAndFilterSurvivors(t *testing.T) {
|
||||
},
|
||||
}, nil
|
||||
}}
|
||||
second := fakeValidator{name: "second", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
second := fakeValidator{name: "second", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
if len(req.CandidateProposal) != 1 {
|
||||
t.Fatalf("expected second validator to see only survivors, got %d", len(req.CandidateProposal))
|
||||
}
|
||||
return validators.Result{ValidatorName: "second", Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: true, ReasonCode: validators.ReasonApproved, Message: "ok"}}}, nil
|
||||
}}
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{first, second}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{first, second}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{
|
||||
{TargetSegmentID: 1, OriginalText: "one", CorrectedText: "ONE"},
|
||||
{TargetSegmentID: 1, OriginalText: "two", CorrectedText: "TWO"},
|
||||
@@ -833,11 +860,11 @@ func TestRunnerValidatorCardinalityErrorStopsPipelineWithPartialProgress(t *test
|
||||
good := fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil
|
||||
}}
|
||||
badValidator := fakeValidator{name: "bad", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
badValidator := fakeValidator{name: "bad", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
// Missing one decision triggers cardinality error.
|
||||
return validators.Result{ValidatorName: "bad", Decisions: nil}, nil
|
||||
}}
|
||||
bad := fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{badValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
bad := fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{badValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}}, nil
|
||||
}}
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"m1": good, "m2": bad}})
|
||||
@@ -856,11 +883,11 @@ func TestRunnerValidatorCardinalityErrorStopsPipelineWithPartialProgress(t *test
|
||||
|
||||
func TestRunnerApplicationSkipAfterValidatorApprovalReported(t *testing.T) {
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "word word"}}}
|
||||
allow := fakeValidator{name: "allow", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
allow := fakeValidator{name: "allow", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
return validators.Result{ValidatorName: "allow", Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: true, ReasonCode: validators.ReasonApproved, Message: "ok"}}}, nil
|
||||
}}
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{allow}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{allow}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "word", CorrectedText: "term"}}, nil
|
||||
}},
|
||||
}})
|
||||
@@ -880,7 +907,7 @@ func TestRunnerApplyConflictAfterTwoInitiallyValidDeletions(t *testing.T) {
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "a b"}}}
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": outcomeModule{
|
||||
fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NonEmptyResultingSegmentValidator{}}},
|
||||
fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NonEmptyResultingSegmentValidator{}}},
|
||||
proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
return contracts.ProposalResult{
|
||||
Proposals: []proposals.CorrectionProposal{
|
||||
@@ -945,14 +972,14 @@ func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contr
|
||||
}
|
||||
|
||||
func TestRunnerLLMValidatorApprovalApplied(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}}}
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}}}
|
||||
llmValidator, err := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
|
||||
if err != nil {
|
||||
t.Fatalf("NewLLMBackedValidator: %v", err)
|
||||
}
|
||||
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil
|
||||
}},
|
||||
}})
|
||||
@@ -971,10 +998,10 @@ func TestRunnerLLMValidatorApprovalApplied(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunnerLLMValidatorRejectionPreventsApplication(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: false, Reason: "reject"}}}}}
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: false, Reason: "reject"}}}}}
|
||||
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil
|
||||
}},
|
||||
}})
|
||||
@@ -996,13 +1023,13 @@ func TestRunnerLLMValidatorRejectionPreventsApplication(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Reason: "bad index"}}}}}
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "99", Approved: true, Reason: "bad index"}}}}}
|
||||
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m1": fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil
|
||||
}},
|
||||
"m2": fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
"m2": fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}}, nil
|
||||
}},
|
||||
}})
|
||||
@@ -1028,13 +1055,13 @@ func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress(
|
||||
|
||||
func TestRunnerReconciledSurvivorsCompleteLaterValidation(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{
|
||||
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
|
||||
{CorrectionIndex: 3, Approved: true, Reason: "ok"},
|
||||
{CorrectionIndex: 99, Approved: true, Reason: "unexpected"},
|
||||
{CorrectionIndex: "1", Approved: true, Reason: "ok"},
|
||||
{CorrectionIndex: "3", Approved: true, Reason: "ok"},
|
||||
{CorrectionIndex: "99", Approved: true, Reason: "unexpected"},
|
||||
}}}}
|
||||
first, _ := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
|
||||
var laterIndices []int
|
||||
later := fakeValidator{name: "later_review", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
later := fakeValidator{name: "later_review", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
decisions := make([]validators.Decision, 0, len(req.CandidateProposal))
|
||||
for _, candidate := range req.CandidateProposal {
|
||||
laterIndices = append(laterIndices, candidate.ProposalIndex)
|
||||
@@ -1047,7 +1074,7 @@ func TestRunnerReconciledSurvivorsCompleteLaterValidation(t *testing.T) {
|
||||
}
|
||||
return validators.Result{ValidatorName: "later_review", Decisions: decisions}, nil
|
||||
}}
|
||||
module := outcomeModule{fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{first, validatormetadata.Wrap(later, validatormetadata.ExecutionClassLLMBacked)}}, proposeResult: func(contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
module := outcomeModule{fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{first, validatormetadata.Wrap(later, validatormetadata.ExecutionClassLLMBacked)}}, proposeResult: func(contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
return contracts.ProposalResult{Proposals: []proposals.CorrectionProposal{
|
||||
{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"},
|
||||
{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"},
|
||||
@@ -1077,7 +1104,7 @@ func TestRunnerLLMValidatorMissingDecisionRejectsBatch(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{}}}}
|
||||
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil
|
||||
}},
|
||||
}})
|
||||
@@ -1094,12 +1121,12 @@ func TestRunnerLLMValidatorMissingDecisionRejectsBatch(t *testing.T) {
|
||||
|
||||
func TestRunnerLLMValidatorDuplicateDecisionRejectsBatch(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{
|
||||
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
|
||||
{CorrectionIndex: 1, Approved: false, Reason: "dup"},
|
||||
{CorrectionIndex: "1", Approved: true, Reason: "ok"},
|
||||
{CorrectionIndex: "1", Approved: false, Reason: "dup"},
|
||||
}}}}
|
||||
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil
|
||||
}},
|
||||
}})
|
||||
@@ -1120,8 +1147,8 @@ func (lenEstimator) EstimateTokens(text string) int { return len(text) }
|
||||
|
||||
func TestRunnerLLMValidatorBatching(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{
|
||||
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}},
|
||||
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}},
|
||||
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}},
|
||||
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}},
|
||||
}}
|
||||
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
|
||||
llmValidator.SetTokenEstimator(lenEstimator{})
|
||||
@@ -1129,7 +1156,7 @@ func TestRunnerLLMValidatorBatching(t *testing.T) {
|
||||
cfg := runnerTestConfig()
|
||||
cfg.Validation.MaxPromptTokens = 260
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyReplaceAll, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyReplaceAll, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{
|
||||
{TargetSegmentID: 1, OriginalText: "alpha", CorrectedText: strings.Repeat("B", 40)},
|
||||
{TargetSegmentID: 1, OriginalText: "gamma", CorrectedText: strings.Repeat("D", 40)},
|
||||
@@ -1154,12 +1181,12 @@ func TestRunnerLLMValidatorBatching(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunnerLLMValidatorDiagnosticsWritten(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "supported by context"}}}}}
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "supported by context"}}}}}
|
||||
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
|
||||
cfg := runnerTestConfig()
|
||||
diagDir := t.TempDir()
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil
|
||||
}},
|
||||
}})
|
||||
@@ -1208,7 +1235,7 @@ type proposalGenerationModule struct {
|
||||
|
||||
func (m proposalGenerationModule) Key() string { return m.key }
|
||||
func (m proposalGenerationModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy }
|
||||
func (m proposalGenerationModule) Validators() []contracts.Validator { return nil }
|
||||
func (m proposalGenerationModule) Validators() []validators.Validator { return nil }
|
||||
func (m proposalGenerationModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
result, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
|
||||
ModuleKey: req.RunSpec.ModuleKey,
|
||||
@@ -1294,10 +1321,10 @@ func TestFatalPartialDecisionsRejectUnknownAndDuplicateIdentity(t *testing.T) {
|
||||
{{ProposalIndex: 8, Approved: true}},
|
||||
{{ProposalIndex: 7, Approved: true}, {ProposalIndex: 7, Approved: true}},
|
||||
} {
|
||||
v := fakeValidator{name: "broken", validateF: func(contracts.ValidationRequest) (validators.Result, error) {
|
||||
v := fakeValidator{name: "broken", validateF: func(validators.Request) (validators.Result, error) {
|
||||
return validators.Result{Decisions: decisions}, origin
|
||||
}}
|
||||
result, err := validateSectionCandidates(context.Background(), validateSectionCandidatesInput{Validators: []contracts.Validator{v}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}})
|
||||
result, err := validateSectionCandidates(context.Background(), validateSectionCandidatesInput{Validators: []validators.Validator{v}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}})
|
||||
if !errors.Is(err, origin) || len(result.decisions) != 0 || len(result.approved) != 0 {
|
||||
t.Fatalf("invalid partial evidence accepted or origin lost: %+v %v", result, err)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user