Validate internal decisions and reconcile exact model references
This commit is contained in:
@@ -25,7 +25,7 @@ func TestFeedbackCollectsIndependentFindingsAndUsableSemanticInput(t *testing.T)
|
||||
return proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: target, OriginalText: original, CorrectedText: replacement}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index}}
|
||||
}
|
||||
var seen []int
|
||||
semantic := validatormetadata.Wrap(fakeValidator{name: "semantic", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
semantic := validatormetadata.Wrap(fakeValidator{name: "semantic", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
var decisions []validators.Decision
|
||||
for _, p := range req.CandidateProposal {
|
||||
seen = append(seen, p.ProposalIndex)
|
||||
@@ -33,7 +33,7 @@ func TestFeedbackCollectsIndependentFindingsAndUsableSemanticInput(t *testing.T)
|
||||
}
|
||||
return validators.Result{Decisions: decisions}, nil
|
||||
}}, validatormetadata.ExecutionClassLLMBacked)
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh teh cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, semantic}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate(0, 1, "cat", "dog"), candidate(1, 0, "teh", "the"), candidate(2, 1, "teh", "the"), candidate(3, 1, "cat", "cat")}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh teh cat"}}}, Validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, semantic}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate(0, 1, "cat", "dog"), candidate(1, 0, "teh", "the"), candidate(2, 1, "teh", "the"), candidate(3, 1, "cat", "cat")}}
|
||||
result, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -58,14 +58,14 @@ func TestFeedbackKeepsUnavailableAndZeroBudgetShortlists(t *testing.T) {
|
||||
for _, budget := range []int{0, 1} {
|
||||
t.Run(fmt.Sprint(budget), func(t *testing.T) {
|
||||
var later int
|
||||
first := fakeValidator{name: "first", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
first := fakeValidator{name: "first", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
var decisions []validators.Decision
|
||||
for _, p := range req.CandidateProposal {
|
||||
decisions = append(decisions, validators.Decision{ProposalIndex: p.ProposalIndex, Disposition: validators.DispositionUnavailable, ReasonCode: "unavailable"})
|
||||
}
|
||||
return validators.Result{Decisions: decisions}, nil
|
||||
}}
|
||||
second := fakeValidator{name: "second", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
second := fakeValidator{name: "second", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
later = len(req.CandidateProposal)
|
||||
var decisions []validators.Decision
|
||||
for _, p := range req.CandidateProposal {
|
||||
@@ -73,7 +73,7 @@ func TestFeedbackKeepsUnavailableAndZeroBudgetShortlists(t *testing.T) {
|
||||
}
|
||||
return validators.Result{Decisions: decisions}, nil
|
||||
}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{first, second}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{first, second}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}}
|
||||
result, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -95,8 +95,8 @@ func TestRepairFindingsUseSafeUnavailableEvidence(t *testing.T) {
|
||||
|
||||
func TestRevalidationArtifactsUseProposalAndAttemptCoordinates(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{
|
||||
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}},
|
||||
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}},
|
||||
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}},
|
||||
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}},
|
||||
}}
|
||||
validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
|
||||
if err != nil {
|
||||
@@ -108,7 +108,7 @@ func TestRevalidationArtifactsUseProposalAndAttemptCoordinates(t *testing.T) {
|
||||
var paths []string
|
||||
for _, index := range []int{2, 3} {
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar_1", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validator}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, ValidationLLMClient: client, Interactions: sink, SectionIndex: §ion, RepairProposalIndex: &index, RepairAttempt: &attempt}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar_1", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{validator}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, ValidationLLMClient: client, Interactions: sink, SectionIndex: §ion, RepairProposalIndex: &index, RepairAttempt: &attempt}
|
||||
result, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -149,11 +149,11 @@ func TestRevalidationArtifactsUseProposalAndAttemptCoordinates(t *testing.T) {
|
||||
|
||||
func TestFeedbackReportsProtectionAndBlankResult(t *testing.T) {
|
||||
identity := glossary.Identity{Collection: "party", ID: "jesters"}
|
||||
protected := fakeValidator{name: "protected_terms", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
protected := fakeValidator{name: "protected_terms", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
return validators.ValidateProtectedTerms(req)
|
||||
}}
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "Jesters", CorrectedText: ""}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "spoken_word", ProtectedTerms: []glossary.Term{{Spelling: "Jesters", Identities: []glossary.Identity{identity}}}, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Jesters"}}}, Validators: []contracts.Validator{protected, validators.NonEmptyResultingSegmentValidator{}}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "spoken_word", ProtectedTerms: []glossary.Term{{Spelling: "Jesters", Identities: []glossary.Identity{identity}}}, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Jesters"}}}, Validators: []validators.Validator{protected, validators.NonEmptyResultingSegmentValidator{}}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}}
|
||||
result, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
Reference in New Issue
Block a user