Remove confidence scoring and refine correction prompts

This commit is contained in:
2026-11-11 17:05:52 +00:00
parent bbf89bb4f5
commit d155bf4cb3
92 changed files with 448 additions and 786 deletions

View File

@@ -5,21 +5,20 @@ import (
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
domain "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
)
func mkCandidate(index int, segID int, orig, corr string, conf float64) proposals.EnrichedCorrectionProposal {
func mkCandidate(index int, segID int, orig, corr string) proposals.EnrichedCorrectionProposal {
return proposals.EnrichedCorrectionProposal{
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: segID, OriginalText: orig, CorrectedText: corr, Confidence: conf},
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: segID, OriginalText: orig, CorrectedText: corr},
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index, ModuleKey: "grammar", ModuleInstance: "grammar_1"},
}
}
func TestEnforceDecisionCardinalitySuccess(t *testing.T) {
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the", 0.9), mkCandidate(1, 1, "recieve", "receive", 0.9)}
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the"), mkCandidate(1, 1, "recieve", "receive")}
decisions := []Decision{{ProposalIndex: 0, Approved: true}, {ProposalIndex: 1, Approved: false}}
if err := EnforceDecisionCardinality(candidates, decisions); err != nil {
t.Fatalf("unexpected error: %v", err)
@@ -27,7 +26,7 @@ func TestEnforceDecisionCardinalitySuccess(t *testing.T) {
}
func TestEnforceDecisionCardinalityMissingDecision(t *testing.T) {
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the", 0.9), mkCandidate(1, 1, "recieve", "receive", 0.9)}
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the"), mkCandidate(1, 1, "recieve", "receive")}
err := EnforceDecisionCardinality(candidates, []Decision{{ProposalIndex: 0, Approved: true}})
if err == nil || !strings.Contains(err.Error(), "missing decision") {
t.Fatalf("expected missing decision error, got %v", err)
@@ -35,7 +34,7 @@ func TestEnforceDecisionCardinalityMissingDecision(t *testing.T) {
}
func TestEnforceDecisionCardinalityDuplicateDecision(t *testing.T) {
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the", 0.9)}
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the")}
err := EnforceDecisionCardinality(candidates, []Decision{{ProposalIndex: 0, Approved: true}, {ProposalIndex: 0, Approved: false}})
if err == nil || !strings.Contains(err.Error(), "duplicate decision") {
t.Fatalf("expected duplicate decision error, got %v", err)
@@ -43,37 +42,19 @@ func TestEnforceDecisionCardinalityDuplicateDecision(t *testing.T) {
}
func TestEnforceDecisionCardinalityUnknownDecision(t *testing.T) {
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the", 0.9)}
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the")}
err := EnforceDecisionCardinality(candidates, []Decision{{ProposalIndex: 99, Approved: true}})
if err == nil || !strings.Contains(err.Error(), "unknown decision") {
t.Fatalf("expected unknown decision error, got %v", err)
}
}
func TestConfidenceThresholdValidator(t *testing.T) {
req := Request{ModuleKey: "grammar", Settings: config.Stage{ConfidenceThreshold: 0.8}, CandidateProposal: []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "teh", "the", 0.9),
mkCandidate(1, 1, "recieve", "receive", 0.7),
}}
res, err := (ConfidenceThresholdValidator{}).Validate(context.Background(), req)
if err != nil {
t.Fatalf("Validate error: %v", err)
}
if !res.Decisions[0].Approved || res.Decisions[0].ReasonCode != ReasonApproved {
t.Fatalf("expected first decision approved, got %+v", res.Decisions[0])
}
if res.Decisions[1].Approved || res.Decisions[1].ReasonCode != ReasonLowConfidence {
t.Fatalf("expected second decision low confidence reject, got %+v", res.Decisions[1])
}
}
func TestProposalShapeValidator(t *testing.T) {
req := Request{CandidateProposal: []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "teh", "the", 0.9),
mkCandidate(1, 0, "teh", "the", 0.9),
mkCandidate(2, 1, "", "the", 0.9),
mkCandidate(3, 1, "teh", "the", 1.5),
mkCandidate(4, 1, " ", " ", 0.9),
mkCandidate(0, 1, "teh", "the"),
mkCandidate(1, 0, "teh", "the"),
mkCandidate(2, 1, "", "the"),
mkCandidate(4, 1, " ", " "),
}}
res, err := (ProposalShapeValidator{}).Validate(context.Background(), req)
if err != nil {
@@ -88,19 +69,16 @@ func TestProposalShapeValidator(t *testing.T) {
if res.Decisions[2].ReasonCode != ReasonEmptyOriginalText {
t.Fatalf("expected empty original rejection, got %+v", res.Decisions[2])
}
if res.Decisions[3].ReasonCode != ReasonInvalidConfidence {
t.Fatalf("expected invalid confidence rejection, got %+v", res.Decisions[3])
}
if !res.Decisions[4].Approved {
t.Fatalf("expected whitespace original approved, got %+v", res.Decisions[4])
if !res.Decisions[3].Approved {
t.Fatalf("expected whitespace original approved, got %+v", res.Decisions[3])
}
}
func TestOriginalTextPresenceValidator(t *testing.T) {
req := Request{WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello world"}}}, CandidateProposal: []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "hello", "hi", 0.9),
mkCandidate(1, 1, "missing", "x", 0.9),
mkCandidate(2, 5, "hello", "hi", 0.9),
mkCandidate(0, 1, "hello", "hi"),
mkCandidate(1, 1, "missing", "x"),
mkCandidate(2, 5, "hello", "hi"),
}}
res, err := (OriginalTextPresenceValidator{}).Validate(context.Background(), req)
if err != nil {
@@ -122,10 +100,10 @@ func TestNonEmptyResultingSegmentValidator(t *testing.T) {
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello world"}}},
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
CandidateProposal: []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "hello", "hi", 0.9),
mkCandidate(1, 1, "hello world", " ", 0.9),
mkCandidate(2, 1, "hello ", "", 0.9),
mkCandidate(3, 1, "hello world", "", 0.9),
mkCandidate(0, 1, "hello", "hi"),
mkCandidate(1, 1, "hello world", " "),
mkCandidate(2, 1, "hello ", ""),
mkCandidate(3, 1, "hello world", ""),
}}
res, err := (NonEmptyResultingSegmentValidator{}).Validate(context.Background(), req)
if err != nil {
@@ -144,8 +122,8 @@ func TestNonEmptyResultingSegmentValidator(t *testing.T) {
func TestNoEffectValidator(t *testing.T) {
req := Request{CandidateProposal: []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "hello", "hello", 0.9),
mkCandidate(1, 1, "hello", "hi", 0.9),
mkCandidate(0, 1, "hello", "hello"),
mkCandidate(1, 1, "hello", "hi"),
}}
res, err := (NoEffectValidator{}).Validate(context.Background(), req)
if err != nil {
@@ -166,8 +144,8 @@ func TestValidateProtectedTerms(t *testing.T) {
ProtectedTerms: domain.Vocabulary(entries, nil),
ModuleKey: "grammar",
CandidateProposal: []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "OpenAI", "Open A I", 0.9),
mkCandidate(1, 1, "teh", "the", 0.9),
mkCandidate(0, 1, "OpenAI", "Open A I"),
mkCandidate(1, 1, "teh", "the"),
},
}
res, err := ValidateProtectedTerms(req)
@@ -185,7 +163,6 @@ func TestValidateProtectedTerms(t *testing.T) {
func TestStableReasonCodes(t *testing.T) {
codes := []string{
ReasonApproved,
ReasonLowConfidence,
ReasonMissingOriginalText,
ReasonMissingTargetSegment,
ReasonEmptyResultingText,
@@ -193,7 +170,6 @@ func TestStableReasonCodes(t *testing.T) {
ReasonProtectedGlossaryTerm,
ReasonInvalidTargetSegment,
ReasonEmptyOriginalText,
ReasonInvalidConfidence,
ReasonValidatorMalformed,
ReasonValidatorInputTooLarge,
}