Remove confidence scoring and refine correction prompts
This commit is contained in:
@@ -2,7 +2,6 @@ package validators
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
|
||||
@@ -21,8 +20,6 @@ func (v ProposalShapeValidator) Validate(_ context.Context, req Request) (Result
|
||||
decisions = append(decisions, rejection(c.ProposalIndex, ReasonInvalidTargetSegment, "proposal target segment id must be positive"))
|
||||
case c.OriginalText == "":
|
||||
decisions = append(decisions, rejection(c.ProposalIndex, ReasonEmptyOriginalText, "proposal original_text must not be empty"))
|
||||
case c.Confidence < 0.0 || c.Confidence > 1.0:
|
||||
decisions = append(decisions, rejection(c.ProposalIndex, ReasonInvalidConfidence, "proposal confidence must be between 0.0 and 1.0"))
|
||||
default:
|
||||
decisions = append(decisions, approval(c.ProposalIndex))
|
||||
}
|
||||
@@ -33,26 +30,6 @@ func (v ProposalShapeValidator) Validate(_ context.Context, req Request) (Result
|
||||
return Result{ValidatorName: v.Name(), Decisions: decisions}, nil
|
||||
}
|
||||
|
||||
type ConfidenceThresholdValidator struct{}
|
||||
|
||||
func (v ConfidenceThresholdValidator) Name() string { return "confidence_threshold" }
|
||||
|
||||
func (v ConfidenceThresholdValidator) Validate(_ context.Context, req Request) (Result, error) {
|
||||
threshold := req.Settings.ConfidenceThreshold
|
||||
decisions := make([]Decision, 0, len(req.CandidateProposal))
|
||||
for _, c := range req.CandidateProposal {
|
||||
if c.Confidence < threshold {
|
||||
decisions = append(decisions, rejection(c.ProposalIndex, ReasonLowConfidence, fmt.Sprintf("confidence %.4f below threshold %.4f", c.Confidence, threshold)))
|
||||
continue
|
||||
}
|
||||
decisions = append(decisions, approval(c.ProposalIndex))
|
||||
}
|
||||
if err := EnforceDecisionCardinality(req.CandidateProposal, decisions); err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
return Result{ValidatorName: v.Name(), Decisions: decisions}, nil
|
||||
}
|
||||
|
||||
type OriginalTextPresenceValidator struct{}
|
||||
|
||||
func (v OriginalTextPresenceValidator) Name() string { return "original_text_presence" }
|
||||
|
||||
@@ -11,7 +11,7 @@ func DefectiveProposalReason(reason string) (outcomes.ReasonCode, bool) {
|
||||
switch reason {
|
||||
case ReasonInvalidTargetSegment, ReasonMissingTargetSegment,
|
||||
ReasonEmptyOriginalText, ReasonMissingOriginalText,
|
||||
ReasonInvalidConfidence, ReasonNoEffect, ReasonEmptyResultingText,
|
||||
ReasonNoEffect, ReasonEmptyResultingText,
|
||||
string(proposals.SkipReasonAmbiguousOriginal),
|
||||
string(proposals.SkipReasonInvalidProposal):
|
||||
return outcomes.ReasonCode(reason), true
|
||||
|
||||
@@ -5,14 +5,14 @@ import "testing"
|
||||
func TestDefectiveProposalReasonSeparatesFilteringFromEditorialRejection(t *testing.T) {
|
||||
for _, reason := range []string{
|
||||
ReasonInvalidTargetSegment, ReasonMissingTargetSegment, ReasonEmptyOriginalText,
|
||||
ReasonMissingOriginalText, ReasonInvalidConfidence, ReasonNoEffect,
|
||||
ReasonMissingOriginalText, ReasonNoEffect,
|
||||
ReasonEmptyResultingText, "ambiguous_original_text",
|
||||
} {
|
||||
if got, ok := DefectiveProposalReason(reason); !ok || string(got) != reason {
|
||||
t.Fatalf("defective proposal %q classified as %q, %t", reason, got, ok)
|
||||
}
|
||||
}
|
||||
for _, reason := range []string{ReasonLowConfidence, ReasonProtectedGlossaryTerm, "editorial_rejection", ReasonValidatorInputTooLarge} {
|
||||
for _, reason := range []string{ReasonProtectedGlossaryTerm, "editorial_rejection", ReasonValidatorInputTooLarge} {
|
||||
if got, ok := DefectiveProposalReason(reason); ok {
|
||||
t.Fatalf("ordinary rejection %q classified as %q", reason, got)
|
||||
}
|
||||
|
||||
@@ -23,7 +23,7 @@ type validationTransport func(*http.Request) (*http.Response, error)
|
||||
func (f validationTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
|
||||
|
||||
func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) {
|
||||
valid := `{"validations":[{"correction_index":1,"approved":true,"confidence":0.9,"reason":"ok"}]}`
|
||||
valid := `{"validations":[{"correction_index":1,"approved":true,"reason":"ok"}]}`
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
responses []string
|
||||
|
||||
@@ -43,10 +43,9 @@ type LLMValidationRequest struct {
|
||||
|
||||
// LLMValidationDecision contains a request-local correction_index, not a proposal index.
|
||||
type LLMValidationDecision struct {
|
||||
CorrectionIndex int `json:"correction_index"`
|
||||
Approved bool `json:"approved"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
Reason string `json:"reason"`
|
||||
CorrectionIndex int `json:"correction_index"`
|
||||
Approved bool `json:"approved"`
|
||||
Reason string `json:"reason"`
|
||||
}
|
||||
|
||||
type LLMValidationResponse struct {
|
||||
|
||||
@@ -48,9 +48,9 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
|
||||
}
|
||||
failed := &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonTransportFailed, Message: "model transport failed"}
|
||||
client := &scriptedValidationClient{replies: []validationReply{
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
|
||||
{completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 1}}, err: failed},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
|
||||
}}
|
||||
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
|
||||
validator.SetTokenEstimator(perItemEstimator{})
|
||||
@@ -73,8 +73,8 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
|
||||
|
||||
survivors := []proposals.EnrichedCorrectionProposal{candidates[0], candidates[2]}
|
||||
nextClient := &scriptedValidationClient{replies: []validationReply{
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
|
||||
}}
|
||||
nextValidator, _ := NewLLMBackedValidator("meaning_reversal_review", LLMValidatorTypeMeaningReversal)
|
||||
nextValidator.SetTokenEstimator(perItemEstimator{})
|
||||
@@ -101,7 +101,7 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
|
||||
|
||||
func TestValidationDomainInvalidAfterRepairIsAnError(t *testing.T) {
|
||||
client := &scriptedValidationClient{replies: []validationReply{{
|
||||
response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: .9, Reason: "ok"}}},
|
||||
response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Reason: "ok"}}},
|
||||
completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 2, StructuralRepairs: 1}},
|
||||
}}}
|
||||
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
|
||||
@@ -116,8 +116,8 @@ func TestValidationDomainInvalidAfterRepairIsAnError(t *testing.T) {
|
||||
func TestValidationExtraDecisionsMergeActualRecoveries(t *testing.T) {
|
||||
client := &scriptedValidationClient{replies: []validationReply{{
|
||||
response: LLMValidationResponse{Validations: []LLMValidationDecision{
|
||||
{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"},
|
||||
{CorrectionIndex: 99, Approved: true, Confidence: .9, Reason: "extra"},
|
||||
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
|
||||
{CorrectionIndex: 99, Approved: true, Reason: "extra"},
|
||||
}},
|
||||
completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 3, TransientRetries: 1, StructuralRepairs: 1}},
|
||||
}}}
|
||||
@@ -137,9 +137,9 @@ func TestValidationExtraDecisionsMergeActualRecoveries(t *testing.T) {
|
||||
func TestValidationFatalFailureKeepsEarlierFactsAndStops(t *testing.T) {
|
||||
cause := errors.New("broken embedded schema")
|
||||
client := &scriptedValidationClient{replies: []validationReply{
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
|
||||
{err: &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "model setup failed", Cause: cause}},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
|
||||
}}
|
||||
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
|
||||
validator.SetTokenEstimator(perItemEstimator{})
|
||||
|
||||
@@ -319,11 +319,6 @@ func reconcileLLMResponse(items []LLMValidationItem, response LLMValidationRespo
|
||||
continue
|
||||
}
|
||||
d := entries[0]
|
||||
if d.Confidence < 0.0 || d.Confidence > 1.0 {
|
||||
result.Decisions = append(result.Decisions, unavailable(item.CorrectionIndex, ReasonInvalidValidationDecision, "required model validation decision had invalid confidence"))
|
||||
result.UnavailableIndices = append(result.UnavailableIndices, item.CorrectionIndex)
|
||||
continue
|
||||
}
|
||||
reasonCode := ReasonApproved
|
||||
if !d.Approved {
|
||||
reasonCode = "llm_rejected"
|
||||
|
||||
@@ -72,7 +72,7 @@ func makeReq(candidates []proposals.EnrichedCorrectionProposal) Request {
|
||||
|
||||
func mk(index int, orig, corr string) proposals.EnrichedCorrectionProposal {
|
||||
return proposals.EnrichedCorrectionProposal{
|
||||
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: orig, CorrectedText: corr, Confidence: 0.9},
|
||||
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: orig, CorrectedText: corr},
|
||||
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index, ModuleKey: "homophones", ModuleInstance: "homophones"},
|
||||
}
|
||||
}
|
||||
@@ -151,8 +151,8 @@ func TestChunkLLMValidationItemsMultipleBatchesStableNoDropNoDup(t *testing.T) {
|
||||
|
||||
func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) {
|
||||
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
|
||||
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
|
||||
{CorrectionIndex: 2, Approved: false, Confidence: 0.95, Reason: "bad"},
|
||||
{CorrectionIndex: 1, Approved: true, Reason: ""},
|
||||
{CorrectionIndex: 2, Approved: false, Reason: "bad"},
|
||||
}}}}
|
||||
v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
|
||||
if err != nil {
|
||||
@@ -164,7 +164,7 @@ func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("validate err: %v", err)
|
||||
}
|
||||
if len(res.Decisions) != 2 || !res.Decisions[0].Approved || res.Decisions[1].Approved {
|
||||
if len(res.Decisions) != 2 || !res.Decisions[0].Approved || res.Decisions[1].Approved || res.Decisions[0].Message != "" || res.Decisions[1].Message != "bad" {
|
||||
t.Fatalf("unexpected decisions: %+v", res.Decisions)
|
||||
}
|
||||
if len(client.calls) != 1 {
|
||||
@@ -206,10 +206,10 @@ func TestLLMBackedValidatorProviderMalformedEnvelopePropagates(t *testing.T) {
|
||||
|
||||
func TestLLMBackedValidatorReconcilesDecisionIdentities(t *testing.T) {
|
||||
approve := func(index int) LLMValidationDecision {
|
||||
return LLMValidationDecision{CorrectionIndex: index, Approved: true, Confidence: .9, Reason: "accepted"}
|
||||
return LLMValidationDecision{CorrectionIndex: index, Approved: true, Reason: "accepted"}
|
||||
}
|
||||
reject := func(index int) LLMValidationDecision {
|
||||
return LLMValidationDecision{CorrectionIndex: index, Confidence: .9, Reason: "editorial concern"}
|
||||
return LLMValidationDecision{CorrectionIndex: index, Reason: "editorial concern"}
|
||||
}
|
||||
type expectedDecision struct {
|
||||
index int
|
||||
@@ -230,7 +230,6 @@ func TestLLMBackedValidatorReconcilesDecisionIdentities(t *testing.T) {
|
||||
{"extra only", []LLMValidationDecision{approve(99), reject(3), approve(1), approve(99), approve(2)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, []int{99, 99}},
|
||||
{"empty", nil, []expectedDecision{{7, false, ReasonMissingValidationDecision}, {11, false, ReasonMissingValidationDecision}, {18, false, ReasonMissingValidationDecision}}, []int{7, 11, 18}, nil},
|
||||
{"reordered", []LLMValidationDecision{reject(3), approve(2), approve(1)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, nil},
|
||||
{"invalid confidence", []LLMValidationDecision{approve(1), {CorrectionIndex: 2, Approved: true, Confidence: 1.2}, reject(3)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonInvalidValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: tc.entries}}}
|
||||
@@ -245,7 +244,7 @@ func TestLLMBackedValidatorReconcilesDecisionIdentities(t *testing.T) {
|
||||
}
|
||||
for i, want := range tc.want {
|
||||
got := result.Decisions[i]
|
||||
if got.ProposalIndex != want.index || got.Approved != want.approved || got.ReasonCode != want.reason || got.DiagnosticArtifactPath != writer.path || (got.Disposition == DispositionUnavailable) != (want.reason == ReasonMissingValidationDecision || want.reason == ReasonDuplicateValidationDecision || want.reason == ReasonInvalidValidationDecision) {
|
||||
if got.ProposalIndex != want.index || got.Approved != want.approved || got.ReasonCode != want.reason || got.DiagnosticArtifactPath != writer.path || (got.Disposition == DispositionUnavailable) != (want.reason == ReasonMissingValidationDecision || want.reason == ReasonDuplicateValidationDecision) {
|
||||
t.Fatalf("decision[%d]=%+v want=%+v", i, got, want)
|
||||
}
|
||||
if got.ReasonCode == "llm_rejected" && got.Message != "editorial concern" {
|
||||
@@ -289,7 +288,7 @@ func equalIndices(a, b []int) bool {
|
||||
|
||||
func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *testing.T) {
|
||||
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
|
||||
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
|
||||
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
|
||||
}}}}
|
||||
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
|
||||
huge := strings.Repeat("gestures ", 200)
|
||||
@@ -300,11 +299,11 @@ func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *tes
|
||||
}},
|
||||
CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
||||
{
|
||||
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: huge, CorrectedText: "Jesters", Confidence: 0.9},
|
||||
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: huge, CorrectedText: "Jesters"},
|
||||
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0, ModuleKey: "homophones", ModuleInstance: "homophones"},
|
||||
},
|
||||
{
|
||||
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 2, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.9},
|
||||
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 2, OriginalText: "gestures", CorrectedText: "Jesters"},
|
||||
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1, ModuleKey: "homophones", ModuleInstance: "homophones"},
|
||||
},
|
||||
},
|
||||
@@ -335,7 +334,7 @@ func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *tes
|
||||
|
||||
func TestLLMBackedValidatorDiagnosticsIncludeSchemaMetadata(t *testing.T) {
|
||||
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
|
||||
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
|
||||
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
|
||||
}}}}
|
||||
writer := &captureValidationDiagnosticsWriter{}
|
||||
v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
|
||||
@@ -374,7 +373,7 @@ func TestEveryValidationTypeSelectsEmbeddedPromptAndVariables(t *testing.T) {
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(string(tc.kind), func(t *testing.T) {
|
||||
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}}}
|
||||
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}}}
|
||||
validator, err := NewLLMBackedValidator(string(tc.kind), tc.kind)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -399,7 +398,7 @@ func TestEveryValidationTypeSelectsEmbeddedPromptAndVariables(t *testing.T) {
|
||||
func TestValidationDiagnosticFailureDoesNotChangeDecision(t *testing.T) {
|
||||
cause := errors.New("diagnostic storage unavailable")
|
||||
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
||||
req.LLMClient = &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 1, Reason: "ok"}}}}}
|
||||
req.LLMClient = &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}}}
|
||||
req.DiagnosticsWriter = &captureValidationDiagnosticsWriter{err: cause}
|
||||
var reported error
|
||||
req.ReportDiagnosticError = func(err error) { reported = err }
|
||||
|
||||
@@ -14,7 +14,6 @@ import (
|
||||
|
||||
const (
|
||||
ReasonApproved = "approved"
|
||||
ReasonLowConfidence = "low_confidence"
|
||||
ReasonMissingOriginalText = "missing_original_text"
|
||||
ReasonMissingTargetSegment = "missing_target_segment"
|
||||
ReasonEmptyResultingText = "empty_resulting_segment"
|
||||
@@ -22,12 +21,10 @@ const (
|
||||
ReasonProtectedGlossaryTerm = "protected_glossary_term"
|
||||
ReasonInvalidTargetSegment = "invalid_target_segment_id"
|
||||
ReasonEmptyOriginalText = "empty_original_text"
|
||||
ReasonInvalidConfidence = "invalid_confidence"
|
||||
ReasonValidatorMalformed = "validator_response_malformed"
|
||||
ReasonValidatorInputTooLarge = "validator_input_too_large"
|
||||
ReasonMissingValidationDecision = "missing_validation_decision"
|
||||
ReasonDuplicateValidationDecision = "duplicate_validation_decision"
|
||||
ReasonInvalidValidationDecision = "invalid_validation_decision"
|
||||
)
|
||||
|
||||
// Request is the runtime input shared by deterministic validators.
|
||||
|
||||
@@ -39,12 +39,11 @@ func TestValidateProtectedTermsAppliesToNonGlossaryModule(t *testing.T) {
|
||||
TargetSegmentID: 1,
|
||||
OriginalText: "Jesters",
|
||||
CorrectedText: "Gestures",
|
||||
Confidence: 0.9,
|
||||
},
|
||||
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0},
|
||||
},
|
||||
{
|
||||
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "Jesters", CorrectedText: "", Confidence: 0.9},
|
||||
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "Jesters", CorrectedText: ""},
|
||||
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1},
|
||||
},
|
||||
},
|
||||
@@ -68,7 +67,6 @@ func TestValidateProtectedTermsGlossaryStageAllowsProtectedTermSwap(t *testing.T
|
||||
TargetSegmentID: 1,
|
||||
OriginalText: "gestures",
|
||||
CorrectedText: "Jesters",
|
||||
Confidence: 0.9,
|
||||
},
|
||||
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0},
|
||||
},
|
||||
@@ -99,7 +97,7 @@ func TestProtectedNameRepetitionRequiresSurvivingIdentityAndSpokenWordScope(t *t
|
||||
req := Request{ModuleKey: tc.module, ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
||||
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: tc.before}}},
|
||||
ProtectedTerms: domain.Vocabulary([]domain.Entry{{ID: "jesters", Name: "Jesters"}, {ID: "lyra", Name: "Lyra"}}, nil),
|
||||
CandidateProposal: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: tc.original, CorrectedText: tc.replacement, Confidence: 1}}},
|
||||
CandidateProposal: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: tc.original, CorrectedText: tc.replacement}}},
|
||||
}
|
||||
result, err := ValidateProtectedTerms(req)
|
||||
if err != nil || len(result.Decisions) != 1 || result.Decisions[0].Approved != tc.allowed {
|
||||
@@ -120,8 +118,7 @@ func TestProtectedNameFindingIdentifiesRemovedAlias(t *testing.T) {
|
||||
ProtectedTerms: domain.Vocabulary([]domain.Entry{{ID: "mira", Name: "Mira", Aliases: []string{"Captain Mira"}}}, nil),
|
||||
CandidateProposal: []proposals.EnrichedCorrectionProposal{{
|
||||
CorrectionProposal: proposals.CorrectionProposal{
|
||||
TargetSegmentID: 1, OriginalText: "Captain Mira", CorrectedText: "the captain", Confidence: 0.9,
|
||||
},
|
||||
TargetSegmentID: 1, OriginalText: "Captain Mira", CorrectedText: "the captain"},
|
||||
}},
|
||||
}
|
||||
result, err := ValidateProtectedTerms(req)
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
package validators
|
||||
|
||||
import (
|
||||
"math"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||
)
|
||||
@@ -50,9 +48,6 @@ func (r Request) Preview(candidate proposals.EnrichedCorrectionProposal) proposa
|
||||
segment = §ion.segments[position]
|
||||
}
|
||||
correction := candidate.CorrectionProposal
|
||||
if r.IndependentEvidence && (math.IsNaN(correction.Confidence) || correction.Confidence < 0 || correction.Confidence > 1) {
|
||||
correction.Confidence = 0.5
|
||||
}
|
||||
preview := proposals.PreviewProposalForSegment(segment, correction, r.ReplacementPolicy)
|
||||
if r.Previews != nil {
|
||||
r.Previews[candidate.ProposalIndex] = preview
|
||||
|
||||
@@ -5,21 +5,20 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||
domain "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||
)
|
||||
|
||||
func mkCandidate(index int, segID int, orig, corr string, conf float64) proposals.EnrichedCorrectionProposal {
|
||||
func mkCandidate(index int, segID int, orig, corr string) proposals.EnrichedCorrectionProposal {
|
||||
return proposals.EnrichedCorrectionProposal{
|
||||
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: segID, OriginalText: orig, CorrectedText: corr, Confidence: conf},
|
||||
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: segID, OriginalText: orig, CorrectedText: corr},
|
||||
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index, ModuleKey: "grammar", ModuleInstance: "grammar_1"},
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnforceDecisionCardinalitySuccess(t *testing.T) {
|
||||
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the", 0.9), mkCandidate(1, 1, "recieve", "receive", 0.9)}
|
||||
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the"), mkCandidate(1, 1, "recieve", "receive")}
|
||||
decisions := []Decision{{ProposalIndex: 0, Approved: true}, {ProposalIndex: 1, Approved: false}}
|
||||
if err := EnforceDecisionCardinality(candidates, decisions); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
@@ -27,7 +26,7 @@ func TestEnforceDecisionCardinalitySuccess(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestEnforceDecisionCardinalityMissingDecision(t *testing.T) {
|
||||
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the", 0.9), mkCandidate(1, 1, "recieve", "receive", 0.9)}
|
||||
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the"), mkCandidate(1, 1, "recieve", "receive")}
|
||||
err := EnforceDecisionCardinality(candidates, []Decision{{ProposalIndex: 0, Approved: true}})
|
||||
if err == nil || !strings.Contains(err.Error(), "missing decision") {
|
||||
t.Fatalf("expected missing decision error, got %v", err)
|
||||
@@ -35,7 +34,7 @@ func TestEnforceDecisionCardinalityMissingDecision(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestEnforceDecisionCardinalityDuplicateDecision(t *testing.T) {
|
||||
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the", 0.9)}
|
||||
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the")}
|
||||
err := EnforceDecisionCardinality(candidates, []Decision{{ProposalIndex: 0, Approved: true}, {ProposalIndex: 0, Approved: false}})
|
||||
if err == nil || !strings.Contains(err.Error(), "duplicate decision") {
|
||||
t.Fatalf("expected duplicate decision error, got %v", err)
|
||||
@@ -43,37 +42,19 @@ func TestEnforceDecisionCardinalityDuplicateDecision(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestEnforceDecisionCardinalityUnknownDecision(t *testing.T) {
|
||||
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the", 0.9)}
|
||||
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the")}
|
||||
err := EnforceDecisionCardinality(candidates, []Decision{{ProposalIndex: 99, Approved: true}})
|
||||
if err == nil || !strings.Contains(err.Error(), "unknown decision") {
|
||||
t.Fatalf("expected unknown decision error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfidenceThresholdValidator(t *testing.T) {
|
||||
req := Request{ModuleKey: "grammar", Settings: config.Stage{ConfidenceThreshold: 0.8}, CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
||||
mkCandidate(0, 1, "teh", "the", 0.9),
|
||||
mkCandidate(1, 1, "recieve", "receive", 0.7),
|
||||
}}
|
||||
res, err := (ConfidenceThresholdValidator{}).Validate(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Validate error: %v", err)
|
||||
}
|
||||
if !res.Decisions[0].Approved || res.Decisions[0].ReasonCode != ReasonApproved {
|
||||
t.Fatalf("expected first decision approved, got %+v", res.Decisions[0])
|
||||
}
|
||||
if res.Decisions[1].Approved || res.Decisions[1].ReasonCode != ReasonLowConfidence {
|
||||
t.Fatalf("expected second decision low confidence reject, got %+v", res.Decisions[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestProposalShapeValidator(t *testing.T) {
|
||||
req := Request{CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
||||
mkCandidate(0, 1, "teh", "the", 0.9),
|
||||
mkCandidate(1, 0, "teh", "the", 0.9),
|
||||
mkCandidate(2, 1, "", "the", 0.9),
|
||||
mkCandidate(3, 1, "teh", "the", 1.5),
|
||||
mkCandidate(4, 1, " ", " ", 0.9),
|
||||
mkCandidate(0, 1, "teh", "the"),
|
||||
mkCandidate(1, 0, "teh", "the"),
|
||||
mkCandidate(2, 1, "", "the"),
|
||||
mkCandidate(4, 1, " ", " "),
|
||||
}}
|
||||
res, err := (ProposalShapeValidator{}).Validate(context.Background(), req)
|
||||
if err != nil {
|
||||
@@ -88,19 +69,16 @@ func TestProposalShapeValidator(t *testing.T) {
|
||||
if res.Decisions[2].ReasonCode != ReasonEmptyOriginalText {
|
||||
t.Fatalf("expected empty original rejection, got %+v", res.Decisions[2])
|
||||
}
|
||||
if res.Decisions[3].ReasonCode != ReasonInvalidConfidence {
|
||||
t.Fatalf("expected invalid confidence rejection, got %+v", res.Decisions[3])
|
||||
}
|
||||
if !res.Decisions[4].Approved {
|
||||
t.Fatalf("expected whitespace original approved, got %+v", res.Decisions[4])
|
||||
if !res.Decisions[3].Approved {
|
||||
t.Fatalf("expected whitespace original approved, got %+v", res.Decisions[3])
|
||||
}
|
||||
}
|
||||
|
||||
func TestOriginalTextPresenceValidator(t *testing.T) {
|
||||
req := Request{WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello world"}}}, CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
||||
mkCandidate(0, 1, "hello", "hi", 0.9),
|
||||
mkCandidate(1, 1, "missing", "x", 0.9),
|
||||
mkCandidate(2, 5, "hello", "hi", 0.9),
|
||||
mkCandidate(0, 1, "hello", "hi"),
|
||||
mkCandidate(1, 1, "missing", "x"),
|
||||
mkCandidate(2, 5, "hello", "hi"),
|
||||
}}
|
||||
res, err := (OriginalTextPresenceValidator{}).Validate(context.Background(), req)
|
||||
if err != nil {
|
||||
@@ -122,10 +100,10 @@ func TestNonEmptyResultingSegmentValidator(t *testing.T) {
|
||||
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello world"}}},
|
||||
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
||||
CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
||||
mkCandidate(0, 1, "hello", "hi", 0.9),
|
||||
mkCandidate(1, 1, "hello world", " ", 0.9),
|
||||
mkCandidate(2, 1, "hello ", "", 0.9),
|
||||
mkCandidate(3, 1, "hello world", "", 0.9),
|
||||
mkCandidate(0, 1, "hello", "hi"),
|
||||
mkCandidate(1, 1, "hello world", " "),
|
||||
mkCandidate(2, 1, "hello ", ""),
|
||||
mkCandidate(3, 1, "hello world", ""),
|
||||
}}
|
||||
res, err := (NonEmptyResultingSegmentValidator{}).Validate(context.Background(), req)
|
||||
if err != nil {
|
||||
@@ -144,8 +122,8 @@ func TestNonEmptyResultingSegmentValidator(t *testing.T) {
|
||||
|
||||
func TestNoEffectValidator(t *testing.T) {
|
||||
req := Request{CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
||||
mkCandidate(0, 1, "hello", "hello", 0.9),
|
||||
mkCandidate(1, 1, "hello", "hi", 0.9),
|
||||
mkCandidate(0, 1, "hello", "hello"),
|
||||
mkCandidate(1, 1, "hello", "hi"),
|
||||
}}
|
||||
res, err := (NoEffectValidator{}).Validate(context.Background(), req)
|
||||
if err != nil {
|
||||
@@ -166,8 +144,8 @@ func TestValidateProtectedTerms(t *testing.T) {
|
||||
ProtectedTerms: domain.Vocabulary(entries, nil),
|
||||
ModuleKey: "grammar",
|
||||
CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
||||
mkCandidate(0, 1, "OpenAI", "Open A I", 0.9),
|
||||
mkCandidate(1, 1, "teh", "the", 0.9),
|
||||
mkCandidate(0, 1, "OpenAI", "Open A I"),
|
||||
mkCandidate(1, 1, "teh", "the"),
|
||||
},
|
||||
}
|
||||
res, err := ValidateProtectedTerms(req)
|
||||
@@ -185,7 +163,6 @@ func TestValidateProtectedTerms(t *testing.T) {
|
||||
func TestStableReasonCodes(t *testing.T) {
|
||||
codes := []string{
|
||||
ReasonApproved,
|
||||
ReasonLowConfidence,
|
||||
ReasonMissingOriginalText,
|
||||
ReasonMissingTargetSegment,
|
||||
ReasonEmptyResultingText,
|
||||
@@ -193,7 +170,6 @@ func TestStableReasonCodes(t *testing.T) {
|
||||
ReasonProtectedGlossaryTerm,
|
||||
ReasonInvalidTargetSegment,
|
||||
ReasonEmptyOriginalText,
|
||||
ReasonInvalidConfidence,
|
||||
ReasonValidatorMalformed,
|
||||
ReasonValidatorInputTooLarge,
|
||||
}
|
||||
|
||||
@@ -50,8 +50,8 @@ func TestApplicationChecksCurrentTextWithSelectedProtection(t *testing.T) {
|
||||
vocabulary := NewProtectedVocabulary(glossary.Vocabulary(protected, active))
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Shield and Aegis"}}}
|
||||
candidates := []proposals.EnrichedCorrectionProposal{
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "Shield", CorrectedText: "Barrier", Confidence: 1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}},
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "egis", CorrectedText: "rmor", Confidence: 1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}},
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "Shield", CorrectedText: "Barrier"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}},
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "egis", CorrectedText: "rmor"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}},
|
||||
}
|
||||
result := proposals.ApplyProposals(transcript, candidates, proposals.ReplacementPolicyRequireUnique, func(before, after string) bool {
|
||||
return vocabulary.AllowsChange(before, after, false)
|
||||
@@ -89,8 +89,7 @@ func TestExemptGlossarySpellingKeepsLongestMatchBoundary(t *testing.T) {
|
||||
terms := glossary.Vocabulary(protected, exclusions)
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: tc.before}}}
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{
|
||||
TargetSegmentID: 1, OriginalText: tc.original, CorrectedText: tc.replacement, Confidence: 1,
|
||||
}}
|
||||
TargetSegmentID: 1, OriginalText: tc.original, CorrectedText: tc.replacement}}
|
||||
policy := proposals.ReplacementPolicyRequireUnique
|
||||
decision, err := ValidateProtectedTerms(Request{ModuleKey: tc.module, WorkingTranscript: transcript, ReplacementPolicy: policy,
|
||||
ProtectedTerms: terms, CandidateProposal: []proposals.EnrichedCorrectionProposal{candidate}})
|
||||
|
||||
Reference in New Issue
Block a user