Remove confidence scoring and refine correction prompts
This commit is contained in:
@@ -5,21 +5,20 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||
domain "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||
)
|
||||
|
||||
func mkCandidate(index int, segID int, orig, corr string, conf float64) proposals.EnrichedCorrectionProposal {
|
||||
func mkCandidate(index int, segID int, orig, corr string) proposals.EnrichedCorrectionProposal {
|
||||
return proposals.EnrichedCorrectionProposal{
|
||||
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: segID, OriginalText: orig, CorrectedText: corr, Confidence: conf},
|
||||
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: segID, OriginalText: orig, CorrectedText: corr},
|
||||
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index, ModuleKey: "grammar", ModuleInstance: "grammar_1"},
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnforceDecisionCardinalitySuccess(t *testing.T) {
|
||||
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the", 0.9), mkCandidate(1, 1, "recieve", "receive", 0.9)}
|
||||
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the"), mkCandidate(1, 1, "recieve", "receive")}
|
||||
decisions := []Decision{{ProposalIndex: 0, Approved: true}, {ProposalIndex: 1, Approved: false}}
|
||||
if err := EnforceDecisionCardinality(candidates, decisions); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
@@ -27,7 +26,7 @@ func TestEnforceDecisionCardinalitySuccess(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestEnforceDecisionCardinalityMissingDecision(t *testing.T) {
|
||||
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the", 0.9), mkCandidate(1, 1, "recieve", "receive", 0.9)}
|
||||
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the"), mkCandidate(1, 1, "recieve", "receive")}
|
||||
err := EnforceDecisionCardinality(candidates, []Decision{{ProposalIndex: 0, Approved: true}})
|
||||
if err == nil || !strings.Contains(err.Error(), "missing decision") {
|
||||
t.Fatalf("expected missing decision error, got %v", err)
|
||||
@@ -35,7 +34,7 @@ func TestEnforceDecisionCardinalityMissingDecision(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestEnforceDecisionCardinalityDuplicateDecision(t *testing.T) {
|
||||
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the", 0.9)}
|
||||
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the")}
|
||||
err := EnforceDecisionCardinality(candidates, []Decision{{ProposalIndex: 0, Approved: true}, {ProposalIndex: 0, Approved: false}})
|
||||
if err == nil || !strings.Contains(err.Error(), "duplicate decision") {
|
||||
t.Fatalf("expected duplicate decision error, got %v", err)
|
||||
@@ -43,37 +42,19 @@ func TestEnforceDecisionCardinalityDuplicateDecision(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestEnforceDecisionCardinalityUnknownDecision(t *testing.T) {
|
||||
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the", 0.9)}
|
||||
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the")}
|
||||
err := EnforceDecisionCardinality(candidates, []Decision{{ProposalIndex: 99, Approved: true}})
|
||||
if err == nil || !strings.Contains(err.Error(), "unknown decision") {
|
||||
t.Fatalf("expected unknown decision error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfidenceThresholdValidator(t *testing.T) {
|
||||
req := Request{ModuleKey: "grammar", Settings: config.Stage{ConfidenceThreshold: 0.8}, CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
||||
mkCandidate(0, 1, "teh", "the", 0.9),
|
||||
mkCandidate(1, 1, "recieve", "receive", 0.7),
|
||||
}}
|
||||
res, err := (ConfidenceThresholdValidator{}).Validate(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("Validate error: %v", err)
|
||||
}
|
||||
if !res.Decisions[0].Approved || res.Decisions[0].ReasonCode != ReasonApproved {
|
||||
t.Fatalf("expected first decision approved, got %+v", res.Decisions[0])
|
||||
}
|
||||
if res.Decisions[1].Approved || res.Decisions[1].ReasonCode != ReasonLowConfidence {
|
||||
t.Fatalf("expected second decision low confidence reject, got %+v", res.Decisions[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestProposalShapeValidator(t *testing.T) {
|
||||
req := Request{CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
||||
mkCandidate(0, 1, "teh", "the", 0.9),
|
||||
mkCandidate(1, 0, "teh", "the", 0.9),
|
||||
mkCandidate(2, 1, "", "the", 0.9),
|
||||
mkCandidate(3, 1, "teh", "the", 1.5),
|
||||
mkCandidate(4, 1, " ", " ", 0.9),
|
||||
mkCandidate(0, 1, "teh", "the"),
|
||||
mkCandidate(1, 0, "teh", "the"),
|
||||
mkCandidate(2, 1, "", "the"),
|
||||
mkCandidate(4, 1, " ", " "),
|
||||
}}
|
||||
res, err := (ProposalShapeValidator{}).Validate(context.Background(), req)
|
||||
if err != nil {
|
||||
@@ -88,19 +69,16 @@ func TestProposalShapeValidator(t *testing.T) {
|
||||
if res.Decisions[2].ReasonCode != ReasonEmptyOriginalText {
|
||||
t.Fatalf("expected empty original rejection, got %+v", res.Decisions[2])
|
||||
}
|
||||
if res.Decisions[3].ReasonCode != ReasonInvalidConfidence {
|
||||
t.Fatalf("expected invalid confidence rejection, got %+v", res.Decisions[3])
|
||||
}
|
||||
if !res.Decisions[4].Approved {
|
||||
t.Fatalf("expected whitespace original approved, got %+v", res.Decisions[4])
|
||||
if !res.Decisions[3].Approved {
|
||||
t.Fatalf("expected whitespace original approved, got %+v", res.Decisions[3])
|
||||
}
|
||||
}
|
||||
|
||||
func TestOriginalTextPresenceValidator(t *testing.T) {
|
||||
req := Request{WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello world"}}}, CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
||||
mkCandidate(0, 1, "hello", "hi", 0.9),
|
||||
mkCandidate(1, 1, "missing", "x", 0.9),
|
||||
mkCandidate(2, 5, "hello", "hi", 0.9),
|
||||
mkCandidate(0, 1, "hello", "hi"),
|
||||
mkCandidate(1, 1, "missing", "x"),
|
||||
mkCandidate(2, 5, "hello", "hi"),
|
||||
}}
|
||||
res, err := (OriginalTextPresenceValidator{}).Validate(context.Background(), req)
|
||||
if err != nil {
|
||||
@@ -122,10 +100,10 @@ func TestNonEmptyResultingSegmentValidator(t *testing.T) {
|
||||
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello world"}}},
|
||||
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
||||
CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
||||
mkCandidate(0, 1, "hello", "hi", 0.9),
|
||||
mkCandidate(1, 1, "hello world", " ", 0.9),
|
||||
mkCandidate(2, 1, "hello ", "", 0.9),
|
||||
mkCandidate(3, 1, "hello world", "", 0.9),
|
||||
mkCandidate(0, 1, "hello", "hi"),
|
||||
mkCandidate(1, 1, "hello world", " "),
|
||||
mkCandidate(2, 1, "hello ", ""),
|
||||
mkCandidate(3, 1, "hello world", ""),
|
||||
}}
|
||||
res, err := (NonEmptyResultingSegmentValidator{}).Validate(context.Background(), req)
|
||||
if err != nil {
|
||||
@@ -144,8 +122,8 @@ func TestNonEmptyResultingSegmentValidator(t *testing.T) {
|
||||
|
||||
func TestNoEffectValidator(t *testing.T) {
|
||||
req := Request{CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
||||
mkCandidate(0, 1, "hello", "hello", 0.9),
|
||||
mkCandidate(1, 1, "hello", "hi", 0.9),
|
||||
mkCandidate(0, 1, "hello", "hello"),
|
||||
mkCandidate(1, 1, "hello", "hi"),
|
||||
}}
|
||||
res, err := (NoEffectValidator{}).Validate(context.Background(), req)
|
||||
if err != nil {
|
||||
@@ -166,8 +144,8 @@ func TestValidateProtectedTerms(t *testing.T) {
|
||||
ProtectedTerms: domain.Vocabulary(entries, nil),
|
||||
ModuleKey: "grammar",
|
||||
CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
||||
mkCandidate(0, 1, "OpenAI", "Open A I", 0.9),
|
||||
mkCandidate(1, 1, "teh", "the", 0.9),
|
||||
mkCandidate(0, 1, "OpenAI", "Open A I"),
|
||||
mkCandidate(1, 1, "teh", "the"),
|
||||
},
|
||||
}
|
||||
res, err := ValidateProtectedTerms(req)
|
||||
@@ -185,7 +163,6 @@ func TestValidateProtectedTerms(t *testing.T) {
|
||||
func TestStableReasonCodes(t *testing.T) {
|
||||
codes := []string{
|
||||
ReasonApproved,
|
||||
ReasonLowConfidence,
|
||||
ReasonMissingOriginalText,
|
||||
ReasonMissingTargetSegment,
|
||||
ReasonEmptyResultingText,
|
||||
@@ -193,7 +170,6 @@ func TestStableReasonCodes(t *testing.T) {
|
||||
ReasonProtectedGlossaryTerm,
|
||||
ReasonInvalidTargetSegment,
|
||||
ReasonEmptyOriginalText,
|
||||
ReasonInvalidConfidence,
|
||||
ReasonValidatorMalformed,
|
||||
ReasonValidatorInputTooLarge,
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user