Validate internal decisions and reconcile exact model references

This commit is contained in:
2026-11-16 14:11:02 +00:00
parent 475557e4de
commit 0c144fd5a5
46 changed files with 544 additions and 376 deletions

View File

@@ -104,7 +104,7 @@ func TestValidatorFailureCancelsActiveAndPendingProviderWork(t *testing.T) {
}, &out)
return validators.Result{}, err
}}
module := fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{validator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
module := fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{validator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "hello", CorrectedText: "Hello"}}, nil
}}
sections := []chunking.Section{}

View File

@@ -1,48 +1,19 @@
package runner
import (
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
)
// feedbackCandidates submits only candidates for which the validator has usable evidence.
func feedbackCandidates(input validateSectionCandidatesInput, validator contracts.Validator, noEffect map[int]bool, evidence contracts.ValidationRequest) []proposals.EnrichedCorrectionProposal {
func feedbackCandidates(input validateSectionCandidatesInput, validatorName string, modelDriven bool, noEffect map[int]bool, evidence validators.Request) []proposals.EnrichedCorrectionProposal {
var result []proposals.EnrichedCorrectionProposal
for _, candidate := range input.SectionEnriched {
if noEffect[candidate.ProposalIndex] {
continue
}
usableSpan := candidate.OriginalText != ""
target := candidate.TargetSegmentID > 0
var applicable bool
switch validator.Name() {
case "proposal_shape":
applicable = true
case "no_effect":
applicable = usableSpan
case "original_text_presence":
applicable = usableSpan && target
case "protected_terms":
applicable = usableSpan
if input.Spec.ModuleKey == modulecatalog.KeyGlossary || input.Spec.ModuleKey == modulecatalog.KeySpokenWord {
preview := evidence.Preview(candidate)
applicable = preview.Applicable || preview.SkipReason == proposals.SkipReasonEmptyResultingText
}
case "non_empty_resulting_segment":
preview := evidence.Preview(candidate)
applicable = preview.Applicable || preview.SkipReason == proposals.SkipReasonEmptyResultingText
default:
if validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked {
applicable = evidence.Preview(candidate).Applicable
} else {
applicable = true
}
}
if applicable {
if evidence.AcceptsIndependentCandidate(validatorName, modelDriven, candidate) {
result = append(result, candidate)
}
}

View File

@@ -25,7 +25,7 @@ func TestFeedbackCollectsIndependentFindingsAndUsableSemanticInput(t *testing.T)
return proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: target, OriginalText: original, CorrectedText: replacement}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index}}
}
var seen []int
semantic := validatormetadata.Wrap(fakeValidator{name: "semantic", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
semantic := validatormetadata.Wrap(fakeValidator{name: "semantic", validateF: func(req validators.Request) (validators.Result, error) {
var decisions []validators.Decision
for _, p := range req.CandidateProposal {
seen = append(seen, p.ProposalIndex)
@@ -33,7 +33,7 @@ func TestFeedbackCollectsIndependentFindingsAndUsableSemanticInput(t *testing.T)
}
return validators.Result{Decisions: decisions}, nil
}}, validatormetadata.ExecutionClassLLMBacked)
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh teh cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, semantic}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate(0, 1, "cat", "dog"), candidate(1, 0, "teh", "the"), candidate(2, 1, "teh", "the"), candidate(3, 1, "cat", "cat")}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh teh cat"}}}, Validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, semantic}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate(0, 1, "cat", "dog"), candidate(1, 0, "teh", "the"), candidate(2, 1, "teh", "the"), candidate(3, 1, "cat", "cat")}}
result, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
@@ -58,14 +58,14 @@ func TestFeedbackKeepsUnavailableAndZeroBudgetShortlists(t *testing.T) {
for _, budget := range []int{0, 1} {
t.Run(fmt.Sprint(budget), func(t *testing.T) {
var later int
first := fakeValidator{name: "first", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
first := fakeValidator{name: "first", validateF: func(req validators.Request) (validators.Result, error) {
var decisions []validators.Decision
for _, p := range req.CandidateProposal {
decisions = append(decisions, validators.Decision{ProposalIndex: p.ProposalIndex, Disposition: validators.DispositionUnavailable, ReasonCode: "unavailable"})
}
return validators.Result{Decisions: decisions}, nil
}}
second := fakeValidator{name: "second", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
second := fakeValidator{name: "second", validateF: func(req validators.Request) (validators.Result, error) {
later = len(req.CandidateProposal)
var decisions []validators.Decision
for _, p := range req.CandidateProposal {
@@ -73,7 +73,7 @@ func TestFeedbackKeepsUnavailableAndZeroBudgetShortlists(t *testing.T) {
}
return validators.Result{Decisions: decisions}, nil
}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{first, second}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{first, second}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}}
result, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
@@ -95,8 +95,8 @@ func TestRepairFindingsUseSafeUnavailableEvidence(t *testing.T) {
func TestRevalidationArtifactsUseProposalAndAttemptCoordinates(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}},
}}
validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
if err != nil {
@@ -108,7 +108,7 @@ func TestRevalidationArtifactsUseProposalAndAttemptCoordinates(t *testing.T) {
var paths []string
for _, index := range []int{2, 3} {
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar_1", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validator}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, ValidationLLMClient: client, Interactions: sink, SectionIndex: &section, RepairProposalIndex: &index, RepairAttempt: &attempt}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar_1", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{validator}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, ValidationLLMClient: client, Interactions: sink, SectionIndex: &section, RepairProposalIndex: &index, RepairAttempt: &attempt}
result, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
@@ -149,11 +149,11 @@ func TestRevalidationArtifactsUseProposalAndAttemptCoordinates(t *testing.T) {
func TestFeedbackReportsProtectionAndBlankResult(t *testing.T) {
identity := glossary.Identity{Collection: "party", ID: "jesters"}
protected := fakeValidator{name: "protected_terms", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
protected := fakeValidator{name: "protected_terms", validateF: func(req validators.Request) (validators.Result, error) {
return validators.ValidateProtectedTerms(req)
}}
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "Jesters", CorrectedText: ""}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "spoken_word", ProtectedTerms: []glossary.Term{{Spelling: "Jesters", Identities: []glossary.Identity{identity}}}, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Jesters"}}}, Validators: []contracts.Validator{protected, validators.NonEmptyResultingSegmentValidator{}}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "spoken_word", ProtectedTerms: []glossary.Term{{Spelling: "Jesters", Identities: []glossary.Identity{identity}}}, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Jesters"}}}, Validators: []validators.Validator{protected, validators.NonEmptyResultingSegmentValidator{}}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}}
result, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)

View File

@@ -2,6 +2,7 @@ package runner
import (
"context"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
@@ -27,7 +28,7 @@ func (m orderedModule) Key() string { return m.spec.ModuleKey }
func (m orderedModule) ReplacementPolicy() proposals.ReplacementPolicy {
return proposals.ReplacementPolicyRequireUnique
}
func (m orderedModule) Validators() []contracts.Validator { return nil }
func (m orderedModule) Validators() []frameworkvalidators.Validator { return nil }
func (m orderedModule) Propose(_ context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
m.seen <- req
original, corrected := "party", "Party"

View File

@@ -24,6 +24,7 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
)
type repairClient struct {
@@ -52,7 +53,7 @@ func TestRepairBranchesRetargetAndPreserveOtherApprovals(t *testing.T) {
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 4}},
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 0, OriginalText: "x", CorrectedText: "y"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 5}},
}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}, {ID: 2, Text: "uh hello"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.NonEmptyResultingSegmentValidator{}}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}, {ID: 2, Text: "uh hello"}}}, Validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.NonEmptyResultingSegmentValidator{}}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{}}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
@@ -92,7 +93,7 @@ func TestRepairRequestCarriesLiteralPreview(t *testing.T) {
Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}},
Policy: tc.policy,
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 81, Text: tc.segment}}},
Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.NonEmptyResultingSegmentValidator{}, rejectUntilRepair()},
Validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.NonEmptyResultingSegmentValidator{}, rejectUntilRepair()},
SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate},
Exchange: &structuredoutput.Exchange{},
}
@@ -131,7 +132,7 @@ func TestRepairRequestRefreshesPreviewAfterRetarget(t *testing.T) {
`{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`,
}}
var validatedIDs []int
reject := fakeValidator{name: "editorial_review", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
reject := fakeValidator{name: "editorial_review", validateF: func(req validators.Request) (validators.Result, error) {
validatedIDs = append(validatedIDs, req.CandidateProposal[0].TargetSegmentID)
return validators.Result{Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Disposition: validators.DispositionRejected, ReasonCode: "llm_rejected", Message: "The context does not support this edit."}}}, nil
}}
@@ -139,9 +140,10 @@ func TestRepairRequestRefreshesPreviewAfterRetarget(t *testing.T) {
Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 2}}},
Policy: proposals.ReplacementPolicyRequireUnique,
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 81, Text: "cat"}, {ID: 104, Text: "uh hello"}}},
Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, reject},
Validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validatormetadata.Wrap(reject, validatormetadata.ExecutionClassDeterministic)},
SectionEnriched: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 81, OriginalText: "cat", CorrectedText: "dog"}}},
Exchange: &structuredoutput.Exchange{},
Interactions: llm.NewDiagnosticsWriter(t.TempDir()),
}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
@@ -154,6 +156,15 @@ func TestRepairRequestRefreshesPreviewAfterRetarget(t *testing.T) {
if len(validatedIDs) != 2 || validatedIDs[0] != 81 || validatedIDs[1] != 104 {
t.Fatalf("validated segment IDs: %v", validatedIDs)
}
for _, decisions := range [][]ValidatorDecisionRecord{initial.decisions, branches[0].Attempts[0].Decisions} {
if len(decisions) == 0 {
t.Fatal("missing decision evidence")
}
record := decisions[len(decisions)-1]
if record.ValidatorName != "editorial_review" || record.ExecutionClass != validatormetadata.ExecutionClassDeterministic {
t.Fatalf("lost execution class override: %+v", record)
}
}
for i, want := range []struct{ before, after string }{{"cat", "dog"}, {"uh hello", "hello"}} {
var payload proposal_generation.RepairRequest
if err := json.Unmarshal([]byte(client.calls[i].Variables["RepairRequestJSON"]), &payload); err != nil {
@@ -183,7 +194,7 @@ func TestRepairBranchTerminalOutcomes(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
client := &repairClient{t: t, outputs: tc.outputs, err: tc.err}
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: tc.budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: tc.budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
@@ -211,7 +222,7 @@ func TestRunnerAppliesRevisionsInOriginalOrder(t *testing.T) {
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":2,"decision":{"action":"revise","correction":{"id":1,"original_text":"cat","corrected_text":"fox"}}}`,
`{"proposal_ref":3,"decision":{"action":"revise","correction":{"id":1,"original_text":"bird","corrected_text":"owl"}}}`,
}}
module := repairModule{outcomeModule{fakeModule: fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, rejectUntilRepair(0)}}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
module := repairModule{outcomeModule{fakeModule: fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, rejectUntilRepair(0)}}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
return contracts.ProposalResult{Proposals: candidates, SegmentReferences: []int{1, 1, 1}, Exchange: &structuredoutput.Exchange{}}, nil
}}}
runner := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}})
@@ -244,7 +255,7 @@ func TestRepairRevalidatesEarlierApprovalAndRetainsUnrelatedApprovalOnFailure(t
}{{"revision", false}, {"provider failure", true}} {
t.Run(tc.name, func(t *testing.T) {
calls := 0
check := fakeValidator{name: "custom", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
check := fakeValidator{name: "custom", validateF: func(req validators.Request) (validators.Result, error) {
var decisions []validators.Decision
for _, candidate := range req.CandidateProposal {
calls++
@@ -256,7 +267,7 @@ func TestRepairRevalidatesEarlierApprovalAndRetainsUnrelatedApprovalOnFailure(t
if tc.failure {
client.err = &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonProviderFailed, Message: "failed"}
}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat bird"}}}, Validators: []contracts.Validator{check, rejectUntilRepair(0)}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat bird"}}}, Validators: []validators.Validator{check, rejectUntilRepair(0)}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{}}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
@@ -279,7 +290,7 @@ func TestRepairRevalidatesEarlierApprovalAndRetainsUnrelatedApprovalOnFailure(t
func TestRepairInterruptionRetainsAttempt(t *testing.T) {
client := &repairClient{t: t, err: &structuredoutput.Failure{Class: structuredoutput.FailureCanceled, Reason: outcomes.ReasonCanceled, Message: "canceled", Cause: context.Canceled}}
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
@@ -297,7 +308,7 @@ func TestRepairArtifactsAreUnique(t *testing.T) {
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "fox"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}},
}
section := 4
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar_1", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: candidates, SectionIndex: &section, Exchange: &structuredoutput.Exchange{}, Interactions: llm.NewDiagnosticsWriter(t.TempDir())}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar_1", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{rejectUntilRepair()}, SectionEnriched: candidates, SectionIndex: &section, Exchange: &structuredoutput.Exchange{}, Interactions: llm.NewDiagnosticsWriter(t.TempDir())}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
@@ -355,7 +366,7 @@ func TestRepairArtifactFailureDoesNotChangeOutcome(t *testing.T) {
}
sink := llm.NewDiagnosticsWriter(path)
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}, Interactions: sink}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}, Interactions: sink}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
@@ -382,8 +393,8 @@ func TestNoEffectAndUnavailableOnlyDoNotStartRepair(t *testing.T) {
}
// rejectUntilRepair supplies a semantic rejection at the external validator seam.
func rejectUntilRepair(preserved ...int) contracts.Validator {
return fakeValidator{name: "editorial_review", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
func rejectUntilRepair(preserved ...int) validators.Validator {
return fakeValidator{name: "editorial_review", validateF: func(req validators.Request) (validators.Result, error) {
var decisions []validators.Decision
for _, p := range req.CandidateProposal {
approved := req.RepairAttempt != nil

View File

@@ -69,13 +69,14 @@ type WorkUnitOutcome struct {
}
type ValidatorDecisionRecord struct {
ValidatorName string `json:"validator_name"`
ProposalIndex int `json:"proposal_index"`
Approved bool `json:"approved"`
Disposition validators.DecisionDisposition `json:"disposition"`
ReasonCode string `json:"reason_code"`
Message string `json:"message"`
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
ExecutionClass validatormetadata.ExecutionClass `json:"-"`
ValidatorName string `json:"validator_name"`
ProposalIndex int `json:"proposal_index"`
Approved bool `json:"approved"`
Disposition validators.DecisionDisposition `json:"disposition"`
ReasonCode string `json:"reason_code"`
Message string `json:"message"`
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
}
type ValidatorRejectedChange struct {
@@ -650,7 +651,7 @@ type validateSectionCandidatesInput struct {
Policy proposals.ReplacementPolicy
WorkingTranscript *schema.Transcript
SectionIndex *int
Validators []contracts.Validator
Validators []validators.Validator
SectionEnriched []proposals.EnrichedCorrectionProposal
ValidationLLMClient contracts.StructuredLLMClient
Interactions structuredoutput.InteractionSink
@@ -673,7 +674,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
eligible := append([]proposals.EnrichedCorrectionProposal(nil), input.SectionEnriched...)
feedback := input.Spec.Settings.LLM.CorrectionRepairAttempts > 0
var blocked, noEffect map[int]bool
evidence := contracts.ValidationRequest{WorkingTranscript: input.WorkingTranscript, Section: input.Section, ReplacementPolicy: input.Policy, IndependentEvidence: feedback}
evidence := validators.Request{WorkingTranscript: input.WorkingTranscript, Section: input.Section, ReplacementPolicy: input.Policy, IndependentEvidence: feedback}
if feedback {
blocked, noEffect = make(map[int]bool), make(map[int]bool)
evidence.Previews = make(map[int]proposals.SegmentPreviewResult)
@@ -686,11 +687,12 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
if err := ctx.Err(); err != nil {
return validateSectionCandidatesResult{decisions: decisions, rejected: rejected, operations: operations, filtered: filtered}, err
}
class := validatormetadata.ClassOf(validator)
submitted := eligible
if feedback {
submitted = feedbackCandidates(input, validator, noEffect, evidence)
submitted = feedbackCandidates(input, validator.Name(), class == validatormetadata.ExecutionClassLLMBacked, noEffect, evidence)
}
vResult, err := validator.Validate(ctx, contracts.ValidationRequest{
vResult, err := validator.Validate(ctx, validators.Request{
StageIndex: &input.Spec.Index, StageID: input.Spec.ID,
WorkingTranscript: input.WorkingTranscript,
Section: evidence.Section, Previews: evidence.Previews,
@@ -728,17 +730,20 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
byIndex[p.ProposalIndex] = p
}
for _, d := range vResult.Decisions {
if feedback && validator.Name() == "original_text_presence" && d.Approved {
preview := evidence.Preview(byIndex[d.ProposalIndex])
if preview.SkipReason == proposals.SkipReasonAmbiguousOriginal {
d.Approved = false
d.Disposition = validators.DispositionRejected
d.ReasonCode = string(proposals.SkipReasonAmbiguousOriginal)
d.Message = "original_text matches multiple locations in the target segment"
}
dispositions := make([]validators.DecisionDisposition, len(vResult.Decisions))
for i, decision := range vResult.Decisions {
disposition, contractErr := decision.EffectiveDisposition()
if contractErr != nil {
return validateSectionCandidatesResult{
decisions: decisions, rejected: rejected, operations: operations, filtered: filtered,
}, fmt.Errorf("validator %q returned inconsistent decisions: %w", validator.Name(), errors.Join(err, contractErr))
}
if feedback && !d.Approved {
dispositions[i] = disposition
}
for i, d := range vResult.Decisions {
disposition := dispositions[i]
approved := disposition == validators.DispositionApproved
if feedback && !approved {
blocked[d.ProposalIndex] = true
if d.ReasonCode == validators.ReasonNoEffect {
noEffect[d.ProposalIndex] = true
@@ -746,18 +751,19 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
}
decisions = append(decisions, ValidatorDecisionRecord{
ValidatorName: validator.Name(),
ExecutionClass: class,
ProposalIndex: d.ProposalIndex,
Approved: d.Approved,
Disposition: d.Disposition,
Approved: approved,
Disposition: disposition,
ReasonCode: d.ReasonCode,
Message: d.Message,
DiagnosticArtifactPath: d.DiagnosticArtifactPath,
})
if d.Approved {
if approved {
nextEligible = append(nextEligible, byIndex[d.ProposalIndex])
continue
}
if d.Disposition != validators.DispositionUnavailable {
if disposition != validators.DispositionUnavailable {
if reason, ok := validators.DefectiveProposalReason(d.ReasonCode); ok {
filtered = append(filtered, outcomes.FilteredProposal{ProposalIndex: d.ProposalIndex, Reason: reason, Count: 1})
}
@@ -771,7 +777,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
TargetSegmentID: p.TargetSegmentID,
OriginalText: p.OriginalText,
CorrectedText: p.CorrectedText,
Disposition: d.Disposition,
Disposition: disposition,
ReasonCode: d.ReasonCode,
Message: d.Message,
})
@@ -882,11 +888,13 @@ func attributeApplyFiltering(sections []chunking.Section, approved []proposals.E
}
}
func reorderValidatorsForPipeline(in []contracts.Validator) ([]contracts.Validator, map[string]int) {
deterministic := make([]contracts.Validator, 0, len(in))
llmBacked := make([]contracts.Validator, 0, len(in))
func reorderValidatorsForPipeline(in []validators.Validator) ([]validators.Validator, map[string]int) {
deterministic := make([]validators.Validator, 0, len(in))
llmBacked := make([]validators.Validator, 0, len(in))
for _, validator := range in {
if validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked {
class := validatormetadata.ClassOf(validator)
validator = validatormetadata.Wrap(validator, class)
if class == validatormetadata.ExecutionClassLLMBacked {
llmBacked = append(llmBacked, validator)
continue
}

View File

@@ -43,7 +43,7 @@ func (f fakeFactory) ModuleForSpec(spec contracts.ModuleRunSpec) (contracts.Tran
type fakeModule struct {
key string
policy proposals.ReplacementPolicy
validators []contracts.Validator
validators []validators.Validator
proposeF func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error)
}
@@ -52,6 +52,33 @@ type outcomeModule struct {
proposeResult func(contracts.ProposalRequest) (contracts.ProposalResult, error)
}
func TestRunnerStopsOnContradictoryValidatorDecision(t *testing.T) {
candidate := proposals.EnrichedCorrectionProposal{
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"},
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 7},
}
contractFailure := errors.New("validator also returned a runtime error")
for _, cause := range []error{nil, contractFailure} {
contradictory := fakeValidator{name: "invalid", validateF: func(validators.Request) (validators.Result, error) {
return validators.Result{Decisions: []validators.Decision{{
ProposalIndex: 7, Approved: true, Disposition: validators.DispositionUnavailable,
}}}, cause
}}
later := fakeValidator{name: "later", validateF: func(validators.Request) (validators.Result, error) {
t.Fatal("candidate advanced past a contradictory decision")
return validators.Result{}, nil
}}
result, err := validateSectionCandidates(context.Background(), validateSectionCandidatesInput{
Validators: []validators.Validator{validators.ProposalShapeValidator{}, contradictory, later},
SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate},
})
if err == nil || !strings.Contains(err.Error(), "contradictory") || (cause != nil && !errors.Is(err, cause)) ||
len(result.approved) != 0 || len(result.decisions) != 1 || result.decisions[0].ValidatorName != "proposal_shape" {
t.Fatalf("contradictory result = %+v (%v)", result, err)
}
}
}
func TestRunnerRecoverableProposalFailureAllowsLaterModule(t *testing.T) {
failed := outcomeModule{fakeModule: fakeModule{key: "first", policy: proposals.ReplacementPolicyRequireUnique}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
return contracts.ProposalResult{Operations: []outcomes.Operation{{Reference: outcomes.Reference{ModuleKey: "first", ModuleInstance: "first", Kind: outcomes.OperationProposal}, FailureReason: outcomes.ReasonTransportFailed}}}, nil
@@ -74,7 +101,7 @@ func TestRunnerRecoverableProposalFailureAllowsLaterModule(t *testing.T) {
func TestRunnerOnlyFullyValidatedCandidatesApplyAndRecoveriesMerge(t *testing.T) {
secondSaw := make([]int, 0)
partial := fakeValidator{name: "partial", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
partial := fakeValidator{name: "partial", validateF: func(req validators.Request) (validators.Result, error) {
if len(req.CandidateProposal) != 2 {
t.Fatalf("expected two candidates after shape filtering, got %d", len(req.CandidateProposal))
}
@@ -86,13 +113,13 @@ func TestRunnerOnlyFullyValidatedCandidatesApplyAndRecoveriesMerge(t *testing.T)
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation}, FailureReason: outcomes.ReasonProviderFailed},
}}, nil
}}
last := fakeValidator{name: "last", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
last := fakeValidator{name: "last", validateF: func(req validators.Request) (validators.Result, error) {
for _, candidate := range req.CandidateProposal {
secondSaw = append(secondSaw, candidate.ProposalIndex)
}
return validators.Result{ValidatorName: "last", Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: true, Disposition: validators.DispositionApproved}}}, nil
}}
module := outcomeModule{fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{validators.ProposalShapeValidator{}, partial, last}}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
module := outcomeModule{fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{validators.ProposalShapeValidator{}, partial, last}}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
return contracts.ProposalResult{Proposals: []proposals.CorrectionProposal{
{TargetSegmentID: 0, OriginalText: "bad", CorrectedText: "x"},
{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"},
@@ -120,10 +147,10 @@ func TestRunnerOnlyFullyValidatedCandidatesApplyAndRecoveriesMerge(t *testing.T)
func TestFatalValidatorNeverReturnsUncheckedCandidates(t *testing.T) {
origin := errors.New("validator setup failed")
proposal := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 7}}
broken := fakeValidator{name: "broken", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
broken := fakeValidator{name: "broken", validateF: func(req validators.Request) (validators.Result, error) {
return validators.Result{Operations: []outcomes.Operation{{Reference: outcomes.Reference{Kind: outcomes.OperationValidation}, FailureReason: outcomes.ReasonSetupFailure}}}, origin
}}
result, err := validateSectionCandidates(context.Background(), validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "m", InstanceName: "m"}, Validators: []contracts.Validator{broken}, SectionEnriched: []proposals.EnrichedCorrectionProposal{proposal}})
result, err := validateSectionCandidates(context.Background(), validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "m", InstanceName: "m"}, Validators: []validators.Validator{broken}, SectionEnriched: []proposals.EnrichedCorrectionProposal{proposal}})
if !errors.Is(err, origin) || len(result.approved) != 0 || len(result.operations) != 1 {
t.Fatalf("fatal result must retain facts without approving candidates: result=%+v err=%v", result, err)
}
@@ -131,7 +158,7 @@ func TestFatalValidatorNeverReturnsUncheckedCandidates(t *testing.T) {
func TestCandidateCaptureIsExplicitAndIncludesFilteredProposals(t *testing.T) {
module := fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique,
validators: []contracts.Validator{validators.ProposalShapeValidator{}},
validators: []validators.Validator{validators.ProposalShapeValidator{}},
proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 0, OriginalText: "bad", CorrectedText: "good"}}, nil
}}
@@ -160,7 +187,7 @@ func (m outcomeModule) Propose(_ context.Context, req contracts.ProposalRequest)
func (m fakeModule) Key() string { return m.key }
func (m fakeModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy }
func (m fakeModule) Validators() []contracts.Validator { return m.validators }
func (m fakeModule) Validators() []validators.Validator { return m.validators }
func (m fakeModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
if m.proposeF == nil {
return contracts.ProposalResult{}, nil
@@ -171,11 +198,11 @@ func (m fakeModule) Propose(ctx context.Context, req contracts.ProposalRequest)
type fakeValidator struct {
name string
validateF func(req contracts.ValidationRequest) (validators.Result, error)
validateF func(req validators.Request) (validators.Result, error)
}
func (v fakeValidator) Name() string { return v.name }
func (v fakeValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (validators.Result, error) {
func (v fakeValidator) Validate(ctx context.Context, req validators.Request) (validators.Result, error) {
_ = ctx
return v.validateF(req)
}
@@ -191,19 +218,19 @@ func (v classifiedFakeValidator) ExecutionClass() validatormetadata.ExecutionCla
func TestReorderValidatorsDeterministicBeforeLLMBacked(t *testing.T) {
llm := classifiedFakeValidator{
fakeValidator: fakeValidator{name: "llm", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
fakeValidator: fakeValidator{name: "llm", validateF: func(req validators.Request) (validators.Result, error) {
return validators.Result{ValidatorName: "llm"}, nil
}},
class: validatormetadata.ExecutionClassLLMBacked,
}
deterministic := classifiedFakeValidator{
fakeValidator: fakeValidator{name: "deterministic", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
fakeValidator: fakeValidator{name: "deterministic", validateF: func(req validators.Request) (validators.Result, error) {
return validators.Result{ValidatorName: "deterministic"}, nil
}},
class: validatormetadata.ExecutionClassDeterministic,
}
ordered, _ := reorderValidatorsForPipeline([]contracts.Validator{llm, deterministic})
ordered, _ := reorderValidatorsForPipeline([]validators.Validator{llm, deterministic})
if len(ordered) != 2 {
t.Fatalf("expected 2 validators, got %d", len(ordered))
}
@@ -213,17 +240,17 @@ func TestReorderValidatorsDeterministicBeforeLLMBacked(t *testing.T) {
}
func TestReorderValidatorsDefaultsUnclassifiedToDeterministic(t *testing.T) {
unclassified := fakeValidator{name: "plain", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
unclassified := fakeValidator{name: "plain", validateF: func(req validators.Request) (validators.Result, error) {
return validators.Result{ValidatorName: "plain"}, nil
}}
llm := classifiedFakeValidator{
fakeValidator: fakeValidator{name: "llm", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
fakeValidator: fakeValidator{name: "llm", validateF: func(req validators.Request) (validators.Result, error) {
return validators.Result{ValidatorName: "llm"}, nil
}},
class: validatormetadata.ExecutionClassLLMBacked,
}
ordered, _ := reorderValidatorsForPipeline([]contracts.Validator{llm, unclassified})
ordered, _ := reorderValidatorsForPipeline([]validators.Validator{llm, unclassified})
if len(ordered) != 2 {
t.Fatalf("expected 2 validators, got %d", len(ordered))
}
@@ -553,7 +580,7 @@ func TestRunnerValidationStartsBeforeAllSectionProposalsComplete(t *testing.T) {
validator := fakeValidator{
name: "capture",
validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
validateF: func(req validators.Request) (validators.Result, error) {
if len(req.CandidateProposal) == 1 && req.CandidateProposal[0].TargetSegmentID == 1 {
select {
case validatorStarted <- struct{}{}:
@@ -577,7 +604,7 @@ func TestRunnerValidationStartsBeforeAllSectionProposalsComplete(t *testing.T) {
"m": fakeModule{
key: "m",
policy: proposals.ReplacementPolicyRequireUnique,
validators: []contracts.Validator{validator},
validators: []validators.Validator{validator},
proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
seg := req.WorkingTranscript.Segments[0]
if req.Section != nil && req.Section.Index == 1 {
@@ -630,7 +657,7 @@ func (c *captureContractStructuredClient) CompleteStructured(ctx context.Context
if target, ok := out.(*validators.LLMValidationResponse); ok {
*target = validators.LLMValidationResponse{
Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
{CorrectionIndex: "1", Approved: true, Reason: "ok"},
},
}
}
@@ -662,7 +689,7 @@ func (c *stageAwareStructuredClient) CompleteStructured(ctx context.Context, req
target := out.(*validators.LLMValidationResponse)
*target = validators.LLMValidationResponse{
Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
{CorrectionIndex: "1", Approved: true, Reason: "ok"},
},
}
return contracts.StructuredCompletionResponse{}, nil
@@ -734,7 +761,7 @@ func TestRunnerRepeatedModuleInstanceNames(t *testing.T) {
func TestRunnerValidatorApprovedProposalApplied(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}}
allowAll := fakeValidator{name: "allow", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
allowAll := fakeValidator{name: "allow", validateF: func(req validators.Request) (validators.Result, error) {
decisions := make([]validators.Decision, len(req.CandidateProposal))
for i, p := range req.CandidateProposal {
decisions[i] = validators.Decision{ProposalIndex: p.ProposalIndex, Approved: true, ReasonCode: validators.ReasonApproved, Message: "approved"}
@@ -745,7 +772,7 @@ func TestRunnerValidatorApprovedProposalApplied(t *testing.T) {
"m": fakeModule{
key: "m",
policy: proposals.ReplacementPolicyRequireUnique,
validators: []contracts.Validator{allowAll},
validators: []validators.Validator{allowAll},
proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil
},
@@ -766,7 +793,7 @@ func TestRunnerValidatorApprovedProposalApplied(t *testing.T) {
func TestRunnerValidatorRejectedProposalNotApplied(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}}
rejectAll := fakeValidator{name: "reject", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
rejectAll := fakeValidator{name: "reject", validateF: func(req validators.Request) (validators.Result, error) {
decisions := make([]validators.Decision, len(req.CandidateProposal))
for i, p := range req.CandidateProposal {
decisions[i] = validators.Decision{ProposalIndex: p.ProposalIndex, Approved: false, ReasonCode: validators.ReasonNoEffect, Message: "rejected"}
@@ -774,7 +801,7 @@ func TestRunnerValidatorRejectedProposalNotApplied(t *testing.T) {
return validators.Result{ValidatorName: "reject", Decisions: decisions}, nil
}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{rejectAll}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{rejectAll}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil
}},
}})
@@ -792,7 +819,7 @@ func TestRunnerValidatorRejectedProposalNotApplied(t *testing.T) {
func TestRunnerMultipleValidatorsRunInOrderAndFilterSurvivors(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "one two"}}}
first := fakeValidator{name: "first", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
first := fakeValidator{name: "first", validateF: func(req validators.Request) (validators.Result, error) {
if len(req.CandidateProposal) != 2 {
t.Fatalf("expected first validator to see 2 candidates, got %d", len(req.CandidateProposal))
}
@@ -804,14 +831,14 @@ func TestRunnerMultipleValidatorsRunInOrderAndFilterSurvivors(t *testing.T) {
},
}, nil
}}
second := fakeValidator{name: "second", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
second := fakeValidator{name: "second", validateF: func(req validators.Request) (validators.Result, error) {
if len(req.CandidateProposal) != 1 {
t.Fatalf("expected second validator to see only survivors, got %d", len(req.CandidateProposal))
}
return validators.Result{ValidatorName: "second", Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: true, ReasonCode: validators.ReasonApproved, Message: "ok"}}}, nil
}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{first, second}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{first, second}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{
{TargetSegmentID: 1, OriginalText: "one", CorrectedText: "ONE"},
{TargetSegmentID: 1, OriginalText: "two", CorrectedText: "TWO"},
@@ -833,11 +860,11 @@ func TestRunnerValidatorCardinalityErrorStopsPipelineWithPartialProgress(t *test
good := fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil
}}
badValidator := fakeValidator{name: "bad", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
badValidator := fakeValidator{name: "bad", validateF: func(req validators.Request) (validators.Result, error) {
// Missing one decision triggers cardinality error.
return validators.Result{ValidatorName: "bad", Decisions: nil}, nil
}}
bad := fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{badValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
bad := fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{badValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}}, nil
}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"m1": good, "m2": bad}})
@@ -856,11 +883,11 @@ func TestRunnerValidatorCardinalityErrorStopsPipelineWithPartialProgress(t *test
func TestRunnerApplicationSkipAfterValidatorApprovalReported(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "word word"}}}
allow := fakeValidator{name: "allow", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
allow := fakeValidator{name: "allow", validateF: func(req validators.Request) (validators.Result, error) {
return validators.Result{ValidatorName: "allow", Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: true, ReasonCode: validators.ReasonApproved, Message: "ok"}}}, nil
}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{allow}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{allow}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "word", CorrectedText: "term"}}, nil
}},
}})
@@ -880,7 +907,7 @@ func TestRunnerApplyConflictAfterTwoInitiallyValidDeletions(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "a b"}}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": outcomeModule{
fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NonEmptyResultingSegmentValidator{}}},
fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NonEmptyResultingSegmentValidator{}}},
proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
return contracts.ProposalResult{
Proposals: []proposals.CorrectionProposal{
@@ -945,14 +972,14 @@ func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contr
}
func TestRunnerLLMValidatorApprovalApplied(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}}}
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}}}
llmValidator, err := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
if err != nil {
t.Fatalf("NewLLMBackedValidator: %v", err)
}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil
}},
}})
@@ -971,10 +998,10 @@ func TestRunnerLLMValidatorApprovalApplied(t *testing.T) {
}
func TestRunnerLLMValidatorRejectionPreventsApplication(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: false, Reason: "reject"}}}}}
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: false, Reason: "reject"}}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil
}},
}})
@@ -996,13 +1023,13 @@ func TestRunnerLLMValidatorRejectionPreventsApplication(t *testing.T) {
}
func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Reason: "bad index"}}}}}
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "99", Approved: true, Reason: "bad index"}}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m1": fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil
}},
"m2": fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
"m2": fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}}, nil
}},
}})
@@ -1028,13 +1055,13 @@ func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress(
func TestRunnerReconciledSurvivorsCompleteLaterValidation(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
{CorrectionIndex: 3, Approved: true, Reason: "ok"},
{CorrectionIndex: 99, Approved: true, Reason: "unexpected"},
{CorrectionIndex: "1", Approved: true, Reason: "ok"},
{CorrectionIndex: "3", Approved: true, Reason: "ok"},
{CorrectionIndex: "99", Approved: true, Reason: "unexpected"},
}}}}
first, _ := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
var laterIndices []int
later := fakeValidator{name: "later_review", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
later := fakeValidator{name: "later_review", validateF: func(req validators.Request) (validators.Result, error) {
decisions := make([]validators.Decision, 0, len(req.CandidateProposal))
for _, candidate := range req.CandidateProposal {
laterIndices = append(laterIndices, candidate.ProposalIndex)
@@ -1047,7 +1074,7 @@ func TestRunnerReconciledSurvivorsCompleteLaterValidation(t *testing.T) {
}
return validators.Result{ValidatorName: "later_review", Decisions: decisions}, nil
}}
module := outcomeModule{fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{first, validatormetadata.Wrap(later, validatormetadata.ExecutionClassLLMBacked)}}, proposeResult: func(contracts.ProposalRequest) (contracts.ProposalResult, error) {
module := outcomeModule{fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{first, validatormetadata.Wrap(later, validatormetadata.ExecutionClassLLMBacked)}}, proposeResult: func(contracts.ProposalRequest) (contracts.ProposalResult, error) {
return contracts.ProposalResult{Proposals: []proposals.CorrectionProposal{
{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"},
{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"},
@@ -1077,7 +1104,7 @@ func TestRunnerLLMValidatorMissingDecisionRejectsBatch(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil
}},
}})
@@ -1094,12 +1121,12 @@ func TestRunnerLLMValidatorMissingDecisionRejectsBatch(t *testing.T) {
func TestRunnerLLMValidatorDuplicateDecisionRejectsBatch(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
{CorrectionIndex: 1, Approved: false, Reason: "dup"},
{CorrectionIndex: "1", Approved: true, Reason: "ok"},
{CorrectionIndex: "1", Approved: false, Reason: "dup"},
}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil
}},
}})
@@ -1120,8 +1147,8 @@ func (lenEstimator) EstimateTokens(text string) int { return len(text) }
func TestRunnerLLMValidatorBatching(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}},
}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
llmValidator.SetTokenEstimator(lenEstimator{})
@@ -1129,7 +1156,7 @@ func TestRunnerLLMValidatorBatching(t *testing.T) {
cfg := runnerTestConfig()
cfg.Validation.MaxPromptTokens = 260
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyReplaceAll, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyReplaceAll, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{
{TargetSegmentID: 1, OriginalText: "alpha", CorrectedText: strings.Repeat("B", 40)},
{TargetSegmentID: 1, OriginalText: "gamma", CorrectedText: strings.Repeat("D", 40)},
@@ -1154,12 +1181,12 @@ func TestRunnerLLMValidatorBatching(t *testing.T) {
}
func TestRunnerLLMValidatorDiagnosticsWritten(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "supported by context"}}}}}
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "supported by context"}}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
cfg := runnerTestConfig()
diagDir := t.TempDir()
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil
}},
}})
@@ -1208,7 +1235,7 @@ type proposalGenerationModule struct {
func (m proposalGenerationModule) Key() string { return m.key }
func (m proposalGenerationModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy }
func (m proposalGenerationModule) Validators() []contracts.Validator { return nil }
func (m proposalGenerationModule) Validators() []validators.Validator { return nil }
func (m proposalGenerationModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
result, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
ModuleKey: req.RunSpec.ModuleKey,
@@ -1294,10 +1321,10 @@ func TestFatalPartialDecisionsRejectUnknownAndDuplicateIdentity(t *testing.T) {
{{ProposalIndex: 8, Approved: true}},
{{ProposalIndex: 7, Approved: true}, {ProposalIndex: 7, Approved: true}},
} {
v := fakeValidator{name: "broken", validateF: func(contracts.ValidationRequest) (validators.Result, error) {
v := fakeValidator{name: "broken", validateF: func(validators.Request) (validators.Result, error) {
return validators.Result{Decisions: decisions}, origin
}}
result, err := validateSectionCandidates(context.Background(), validateSectionCandidatesInput{Validators: []contracts.Validator{v}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}})
result, err := validateSectionCandidates(context.Background(), validateSectionCandidatesInput{Validators: []validators.Validator{v}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}})
if !errors.Is(err, origin) || len(result.decisions) != 0 || len(result.approved) != 0 {
t.Fatalf("invalid partial evidence accepted or origin lost: %+v %v", result, err)
}