Remove confidence scoring and refine correction prompts

This commit is contained in:
2026-11-11 17:05:52 +00:00
parent bbf89bb4f5
commit d155bf4cb3
92 changed files with 448 additions and 786 deletions

View File

@@ -2,7 +2,6 @@ package validators
import (
"context"
"fmt"
"strings"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
@@ -21,8 +20,6 @@ func (v ProposalShapeValidator) Validate(_ context.Context, req Request) (Result
decisions = append(decisions, rejection(c.ProposalIndex, ReasonInvalidTargetSegment, "proposal target segment id must be positive"))
case c.OriginalText == "":
decisions = append(decisions, rejection(c.ProposalIndex, ReasonEmptyOriginalText, "proposal original_text must not be empty"))
case c.Confidence < 0.0 || c.Confidence > 1.0:
decisions = append(decisions, rejection(c.ProposalIndex, ReasonInvalidConfidence, "proposal confidence must be between 0.0 and 1.0"))
default:
decisions = append(decisions, approval(c.ProposalIndex))
}
@@ -33,26 +30,6 @@ func (v ProposalShapeValidator) Validate(_ context.Context, req Request) (Result
return Result{ValidatorName: v.Name(), Decisions: decisions}, nil
}
type ConfidenceThresholdValidator struct{}
func (v ConfidenceThresholdValidator) Name() string { return "confidence_threshold" }
func (v ConfidenceThresholdValidator) Validate(_ context.Context, req Request) (Result, error) {
threshold := req.Settings.ConfidenceThreshold
decisions := make([]Decision, 0, len(req.CandidateProposal))
for _, c := range req.CandidateProposal {
if c.Confidence < threshold {
decisions = append(decisions, rejection(c.ProposalIndex, ReasonLowConfidence, fmt.Sprintf("confidence %.4f below threshold %.4f", c.Confidence, threshold)))
continue
}
decisions = append(decisions, approval(c.ProposalIndex))
}
if err := EnforceDecisionCardinality(req.CandidateProposal, decisions); err != nil {
return Result{}, err
}
return Result{ValidatorName: v.Name(), Decisions: decisions}, nil
}
type OriginalTextPresenceValidator struct{}
func (v OriginalTextPresenceValidator) Name() string { return "original_text_presence" }

View File

@@ -11,7 +11,7 @@ func DefectiveProposalReason(reason string) (outcomes.ReasonCode, bool) {
switch reason {
case ReasonInvalidTargetSegment, ReasonMissingTargetSegment,
ReasonEmptyOriginalText, ReasonMissingOriginalText,
ReasonInvalidConfidence, ReasonNoEffect, ReasonEmptyResultingText,
ReasonNoEffect, ReasonEmptyResultingText,
string(proposals.SkipReasonAmbiguousOriginal),
string(proposals.SkipReasonInvalidProposal):
return outcomes.ReasonCode(reason), true

View File

@@ -5,14 +5,14 @@ import "testing"
func TestDefectiveProposalReasonSeparatesFilteringFromEditorialRejection(t *testing.T) {
for _, reason := range []string{
ReasonInvalidTargetSegment, ReasonMissingTargetSegment, ReasonEmptyOriginalText,
ReasonMissingOriginalText, ReasonInvalidConfidence, ReasonNoEffect,
ReasonMissingOriginalText, ReasonNoEffect,
ReasonEmptyResultingText, "ambiguous_original_text",
} {
if got, ok := DefectiveProposalReason(reason); !ok || string(got) != reason {
t.Fatalf("defective proposal %q classified as %q, %t", reason, got, ok)
}
}
for _, reason := range []string{ReasonLowConfidence, ReasonProtectedGlossaryTerm, "editorial_rejection", ReasonValidatorInputTooLarge} {
for _, reason := range []string{ReasonProtectedGlossaryTerm, "editorial_rejection", ReasonValidatorInputTooLarge} {
if got, ok := DefectiveProposalReason(reason); ok {
t.Fatalf("ordinary rejection %q classified as %q", reason, got)
}

View File

@@ -23,7 +23,7 @@ type validationTransport func(*http.Request) (*http.Response, error)
func (f validationTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) {
valid := `{"validations":[{"correction_index":1,"approved":true,"confidence":0.9,"reason":"ok"}]}`
valid := `{"validations":[{"correction_index":1,"approved":true,"reason":"ok"}]}`
for _, tc := range []struct {
name string
responses []string

View File

@@ -43,10 +43,9 @@ type LLMValidationRequest struct {
// LLMValidationDecision contains a request-local correction_index, not a proposal index.
type LLMValidationDecision struct {
CorrectionIndex int `json:"correction_index"`
Approved bool `json:"approved"`
Confidence float64 `json:"confidence"`
Reason string `json:"reason"`
CorrectionIndex int `json:"correction_index"`
Approved bool `json:"approved"`
Reason string `json:"reason"`
}
type LLMValidationResponse struct {

View File

@@ -48,9 +48,9 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
}
failed := &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonTransportFailed, Message: "model transport failed"}
client := &scriptedValidationClient{replies: []validationReply{
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
{completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 1}}, err: failed},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
}}
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
validator.SetTokenEstimator(perItemEstimator{})
@@ -73,8 +73,8 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
survivors := []proposals.EnrichedCorrectionProposal{candidates[0], candidates[2]}
nextClient := &scriptedValidationClient{replies: []validationReply{
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
}}
nextValidator, _ := NewLLMBackedValidator("meaning_reversal_review", LLMValidatorTypeMeaningReversal)
nextValidator.SetTokenEstimator(perItemEstimator{})
@@ -101,7 +101,7 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
func TestValidationDomainInvalidAfterRepairIsAnError(t *testing.T) {
client := &scriptedValidationClient{replies: []validationReply{{
response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: .9, Reason: "ok"}}},
response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Reason: "ok"}}},
completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 2, StructuralRepairs: 1}},
}}}
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
@@ -116,8 +116,8 @@ func TestValidationDomainInvalidAfterRepairIsAnError(t *testing.T) {
func TestValidationExtraDecisionsMergeActualRecoveries(t *testing.T) {
client := &scriptedValidationClient{replies: []validationReply{{
response: LLMValidationResponse{Validations: []LLMValidationDecision{
{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"},
{CorrectionIndex: 99, Approved: true, Confidence: .9, Reason: "extra"},
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
{CorrectionIndex: 99, Approved: true, Reason: "extra"},
}},
completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 3, TransientRetries: 1, StructuralRepairs: 1}},
}}}
@@ -137,9 +137,9 @@ func TestValidationExtraDecisionsMergeActualRecoveries(t *testing.T) {
func TestValidationFatalFailureKeepsEarlierFactsAndStops(t *testing.T) {
cause := errors.New("broken embedded schema")
client := &scriptedValidationClient{replies: []validationReply{
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
{err: &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "model setup failed", Cause: cause}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
}}
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
validator.SetTokenEstimator(perItemEstimator{})

View File

@@ -319,11 +319,6 @@ func reconcileLLMResponse(items []LLMValidationItem, response LLMValidationRespo
continue
}
d := entries[0]
if d.Confidence < 0.0 || d.Confidence > 1.0 {
result.Decisions = append(result.Decisions, unavailable(item.CorrectionIndex, ReasonInvalidValidationDecision, "required model validation decision had invalid confidence"))
result.UnavailableIndices = append(result.UnavailableIndices, item.CorrectionIndex)
continue
}
reasonCode := ReasonApproved
if !d.Approved {
reasonCode = "llm_rejected"

View File

@@ -72,7 +72,7 @@ func makeReq(candidates []proposals.EnrichedCorrectionProposal) Request {
func mk(index int, orig, corr string) proposals.EnrichedCorrectionProposal {
return proposals.EnrichedCorrectionProposal{
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: orig, CorrectedText: corr, Confidence: 0.9},
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: orig, CorrectedText: corr},
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index, ModuleKey: "homophones", ModuleInstance: "homophones"},
}
}
@@ -151,8 +151,8 @@ func TestChunkLLMValidationItemsMultipleBatchesStableNoDropNoDup(t *testing.T) {
func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 2, Approved: false, Confidence: 0.95, Reason: "bad"},
{CorrectionIndex: 1, Approved: true, Reason: ""},
{CorrectionIndex: 2, Approved: false, Reason: "bad"},
}}}}
v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
if err != nil {
@@ -164,7 +164,7 @@ func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) {
if err != nil {
t.Fatalf("validate err: %v", err)
}
if len(res.Decisions) != 2 || !res.Decisions[0].Approved || res.Decisions[1].Approved {
if len(res.Decisions) != 2 || !res.Decisions[0].Approved || res.Decisions[1].Approved || res.Decisions[0].Message != "" || res.Decisions[1].Message != "bad" {
t.Fatalf("unexpected decisions: %+v", res.Decisions)
}
if len(client.calls) != 1 {
@@ -206,10 +206,10 @@ func TestLLMBackedValidatorProviderMalformedEnvelopePropagates(t *testing.T) {
func TestLLMBackedValidatorReconcilesDecisionIdentities(t *testing.T) {
approve := func(index int) LLMValidationDecision {
return LLMValidationDecision{CorrectionIndex: index, Approved: true, Confidence: .9, Reason: "accepted"}
return LLMValidationDecision{CorrectionIndex: index, Approved: true, Reason: "accepted"}
}
reject := func(index int) LLMValidationDecision {
return LLMValidationDecision{CorrectionIndex: index, Confidence: .9, Reason: "editorial concern"}
return LLMValidationDecision{CorrectionIndex: index, Reason: "editorial concern"}
}
type expectedDecision struct {
index int
@@ -230,7 +230,6 @@ func TestLLMBackedValidatorReconcilesDecisionIdentities(t *testing.T) {
{"extra only", []LLMValidationDecision{approve(99), reject(3), approve(1), approve(99), approve(2)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, []int{99, 99}},
{"empty", nil, []expectedDecision{{7, false, ReasonMissingValidationDecision}, {11, false, ReasonMissingValidationDecision}, {18, false, ReasonMissingValidationDecision}}, []int{7, 11, 18}, nil},
{"reordered", []LLMValidationDecision{reject(3), approve(2), approve(1)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, nil},
{"invalid confidence", []LLMValidationDecision{approve(1), {CorrectionIndex: 2, Approved: true, Confidence: 1.2}, reject(3)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonInvalidValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
} {
t.Run(tc.name, func(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: tc.entries}}}
@@ -245,7 +244,7 @@ func TestLLMBackedValidatorReconcilesDecisionIdentities(t *testing.T) {
}
for i, want := range tc.want {
got := result.Decisions[i]
if got.ProposalIndex != want.index || got.Approved != want.approved || got.ReasonCode != want.reason || got.DiagnosticArtifactPath != writer.path || (got.Disposition == DispositionUnavailable) != (want.reason == ReasonMissingValidationDecision || want.reason == ReasonDuplicateValidationDecision || want.reason == ReasonInvalidValidationDecision) {
if got.ProposalIndex != want.index || got.Approved != want.approved || got.ReasonCode != want.reason || got.DiagnosticArtifactPath != writer.path || (got.Disposition == DispositionUnavailable) != (want.reason == ReasonMissingValidationDecision || want.reason == ReasonDuplicateValidationDecision) {
t.Fatalf("decision[%d]=%+v want=%+v", i, got, want)
}
if got.ReasonCode == "llm_rejected" && got.Message != "editorial concern" {
@@ -289,7 +288,7 @@ func equalIndices(a, b []int) bool {
func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
}}}}
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
huge := strings.Repeat("gestures ", 200)
@@ -300,11 +299,11 @@ func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *tes
}},
CandidateProposal: []proposals.EnrichedCorrectionProposal{
{
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: huge, CorrectedText: "Jesters", Confidence: 0.9},
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: huge, CorrectedText: "Jesters"},
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0, ModuleKey: "homophones", ModuleInstance: "homophones"},
},
{
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 2, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.9},
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 2, OriginalText: "gestures", CorrectedText: "Jesters"},
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1, ModuleKey: "homophones", ModuleInstance: "homophones"},
},
},
@@ -335,7 +334,7 @@ func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *tes
func TestLLMBackedValidatorDiagnosticsIncludeSchemaMetadata(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
}}}}
writer := &captureValidationDiagnosticsWriter{}
v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
@@ -374,7 +373,7 @@ func TestEveryValidationTypeSelectsEmbeddedPromptAndVariables(t *testing.T) {
}
for _, tc := range cases {
t.Run(string(tc.kind), func(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: .9, Reason: "ok"}}}}}
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}}}
validator, err := NewLLMBackedValidator(string(tc.kind), tc.kind)
if err != nil {
t.Fatal(err)
@@ -399,7 +398,7 @@ func TestEveryValidationTypeSelectsEmbeddedPromptAndVariables(t *testing.T) {
func TestValidationDiagnosticFailureDoesNotChangeDecision(t *testing.T) {
cause := errors.New("diagnostic storage unavailable")
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 1, Reason: "ok"}}}}}
req.LLMClient = &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}}}
req.DiagnosticsWriter = &captureValidationDiagnosticsWriter{err: cause}
var reported error
req.ReportDiagnosticError = func(err error) { reported = err }

View File

@@ -14,7 +14,6 @@ import (
const (
ReasonApproved = "approved"
ReasonLowConfidence = "low_confidence"
ReasonMissingOriginalText = "missing_original_text"
ReasonMissingTargetSegment = "missing_target_segment"
ReasonEmptyResultingText = "empty_resulting_segment"
@@ -22,12 +21,10 @@ const (
ReasonProtectedGlossaryTerm = "protected_glossary_term"
ReasonInvalidTargetSegment = "invalid_target_segment_id"
ReasonEmptyOriginalText = "empty_original_text"
ReasonInvalidConfidence = "invalid_confidence"
ReasonValidatorMalformed = "validator_response_malformed"
ReasonValidatorInputTooLarge = "validator_input_too_large"
ReasonMissingValidationDecision = "missing_validation_decision"
ReasonDuplicateValidationDecision = "duplicate_validation_decision"
ReasonInvalidValidationDecision = "invalid_validation_decision"
)
// Request is the runtime input shared by deterministic validators.

View File

@@ -39,12 +39,11 @@ func TestValidateProtectedTermsAppliesToNonGlossaryModule(t *testing.T) {
TargetSegmentID: 1,
OriginalText: "Jesters",
CorrectedText: "Gestures",
Confidence: 0.9,
},
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0},
},
{
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "Jesters", CorrectedText: "", Confidence: 0.9},
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "Jesters", CorrectedText: ""},
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1},
},
},
@@ -68,7 +67,6 @@ func TestValidateProtectedTermsGlossaryStageAllowsProtectedTermSwap(t *testing.T
TargetSegmentID: 1,
OriginalText: "gestures",
CorrectedText: "Jesters",
Confidence: 0.9,
},
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0},
},
@@ -99,7 +97,7 @@ func TestProtectedNameRepetitionRequiresSurvivingIdentityAndSpokenWordScope(t *t
req := Request{ModuleKey: tc.module, ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: tc.before}}},
ProtectedTerms: domain.Vocabulary([]domain.Entry{{ID: "jesters", Name: "Jesters"}, {ID: "lyra", Name: "Lyra"}}, nil),
CandidateProposal: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: tc.original, CorrectedText: tc.replacement, Confidence: 1}}},
CandidateProposal: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: tc.original, CorrectedText: tc.replacement}}},
}
result, err := ValidateProtectedTerms(req)
if err != nil || len(result.Decisions) != 1 || result.Decisions[0].Approved != tc.allowed {
@@ -120,8 +118,7 @@ func TestProtectedNameFindingIdentifiesRemovedAlias(t *testing.T) {
ProtectedTerms: domain.Vocabulary([]domain.Entry{{ID: "mira", Name: "Mira", Aliases: []string{"Captain Mira"}}}, nil),
CandidateProposal: []proposals.EnrichedCorrectionProposal{{
CorrectionProposal: proposals.CorrectionProposal{
TargetSegmentID: 1, OriginalText: "Captain Mira", CorrectedText: "the captain", Confidence: 0.9,
},
TargetSegmentID: 1, OriginalText: "Captain Mira", CorrectedText: "the captain"},
}},
}
result, err := ValidateProtectedTerms(req)

View File

@@ -1,8 +1,6 @@
package validators
import (
"math"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
)
@@ -50,9 +48,6 @@ func (r Request) Preview(candidate proposals.EnrichedCorrectionProposal) proposa
segment = &section.segments[position]
}
correction := candidate.CorrectionProposal
if r.IndependentEvidence && (math.IsNaN(correction.Confidence) || correction.Confidence < 0 || correction.Confidence > 1) {
correction.Confidence = 0.5
}
preview := proposals.PreviewProposalForSegment(segment, correction, r.ReplacementPolicy)
if r.Previews != nil {
r.Previews[candidate.ProposalIndex] = preview

View File

@@ -5,21 +5,20 @@ import (
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
domain "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
)
func mkCandidate(index int, segID int, orig, corr string, conf float64) proposals.EnrichedCorrectionProposal {
func mkCandidate(index int, segID int, orig, corr string) proposals.EnrichedCorrectionProposal {
return proposals.EnrichedCorrectionProposal{
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: segID, OriginalText: orig, CorrectedText: corr, Confidence: conf},
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: segID, OriginalText: orig, CorrectedText: corr},
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index, ModuleKey: "grammar", ModuleInstance: "grammar_1"},
}
}
func TestEnforceDecisionCardinalitySuccess(t *testing.T) {
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the", 0.9), mkCandidate(1, 1, "recieve", "receive", 0.9)}
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the"), mkCandidate(1, 1, "recieve", "receive")}
decisions := []Decision{{ProposalIndex: 0, Approved: true}, {ProposalIndex: 1, Approved: false}}
if err := EnforceDecisionCardinality(candidates, decisions); err != nil {
t.Fatalf("unexpected error: %v", err)
@@ -27,7 +26,7 @@ func TestEnforceDecisionCardinalitySuccess(t *testing.T) {
}
func TestEnforceDecisionCardinalityMissingDecision(t *testing.T) {
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the", 0.9), mkCandidate(1, 1, "recieve", "receive", 0.9)}
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the"), mkCandidate(1, 1, "recieve", "receive")}
err := EnforceDecisionCardinality(candidates, []Decision{{ProposalIndex: 0, Approved: true}})
if err == nil || !strings.Contains(err.Error(), "missing decision") {
t.Fatalf("expected missing decision error, got %v", err)
@@ -35,7 +34,7 @@ func TestEnforceDecisionCardinalityMissingDecision(t *testing.T) {
}
func TestEnforceDecisionCardinalityDuplicateDecision(t *testing.T) {
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the", 0.9)}
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the")}
err := EnforceDecisionCardinality(candidates, []Decision{{ProposalIndex: 0, Approved: true}, {ProposalIndex: 0, Approved: false}})
if err == nil || !strings.Contains(err.Error(), "duplicate decision") {
t.Fatalf("expected duplicate decision error, got %v", err)
@@ -43,37 +42,19 @@ func TestEnforceDecisionCardinalityDuplicateDecision(t *testing.T) {
}
func TestEnforceDecisionCardinalityUnknownDecision(t *testing.T) {
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the", 0.9)}
candidates := []proposals.EnrichedCorrectionProposal{mkCandidate(0, 1, "teh", "the")}
err := EnforceDecisionCardinality(candidates, []Decision{{ProposalIndex: 99, Approved: true}})
if err == nil || !strings.Contains(err.Error(), "unknown decision") {
t.Fatalf("expected unknown decision error, got %v", err)
}
}
func TestConfidenceThresholdValidator(t *testing.T) {
req := Request{ModuleKey: "grammar", Settings: config.Stage{ConfidenceThreshold: 0.8}, CandidateProposal: []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "teh", "the", 0.9),
mkCandidate(1, 1, "recieve", "receive", 0.7),
}}
res, err := (ConfidenceThresholdValidator{}).Validate(context.Background(), req)
if err != nil {
t.Fatalf("Validate error: %v", err)
}
if !res.Decisions[0].Approved || res.Decisions[0].ReasonCode != ReasonApproved {
t.Fatalf("expected first decision approved, got %+v", res.Decisions[0])
}
if res.Decisions[1].Approved || res.Decisions[1].ReasonCode != ReasonLowConfidence {
t.Fatalf("expected second decision low confidence reject, got %+v", res.Decisions[1])
}
}
func TestProposalShapeValidator(t *testing.T) {
req := Request{CandidateProposal: []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "teh", "the", 0.9),
mkCandidate(1, 0, "teh", "the", 0.9),
mkCandidate(2, 1, "", "the", 0.9),
mkCandidate(3, 1, "teh", "the", 1.5),
mkCandidate(4, 1, " ", " ", 0.9),
mkCandidate(0, 1, "teh", "the"),
mkCandidate(1, 0, "teh", "the"),
mkCandidate(2, 1, "", "the"),
mkCandidate(4, 1, " ", " "),
}}
res, err := (ProposalShapeValidator{}).Validate(context.Background(), req)
if err != nil {
@@ -88,19 +69,16 @@ func TestProposalShapeValidator(t *testing.T) {
if res.Decisions[2].ReasonCode != ReasonEmptyOriginalText {
t.Fatalf("expected empty original rejection, got %+v", res.Decisions[2])
}
if res.Decisions[3].ReasonCode != ReasonInvalidConfidence {
t.Fatalf("expected invalid confidence rejection, got %+v", res.Decisions[3])
}
if !res.Decisions[4].Approved {
t.Fatalf("expected whitespace original approved, got %+v", res.Decisions[4])
if !res.Decisions[3].Approved {
t.Fatalf("expected whitespace original approved, got %+v", res.Decisions[3])
}
}
func TestOriginalTextPresenceValidator(t *testing.T) {
req := Request{WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello world"}}}, CandidateProposal: []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "hello", "hi", 0.9),
mkCandidate(1, 1, "missing", "x", 0.9),
mkCandidate(2, 5, "hello", "hi", 0.9),
mkCandidate(0, 1, "hello", "hi"),
mkCandidate(1, 1, "missing", "x"),
mkCandidate(2, 5, "hello", "hi"),
}}
res, err := (OriginalTextPresenceValidator{}).Validate(context.Background(), req)
if err != nil {
@@ -122,10 +100,10 @@ func TestNonEmptyResultingSegmentValidator(t *testing.T) {
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello world"}}},
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
CandidateProposal: []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "hello", "hi", 0.9),
mkCandidate(1, 1, "hello world", " ", 0.9),
mkCandidate(2, 1, "hello ", "", 0.9),
mkCandidate(3, 1, "hello world", "", 0.9),
mkCandidate(0, 1, "hello", "hi"),
mkCandidate(1, 1, "hello world", " "),
mkCandidate(2, 1, "hello ", ""),
mkCandidate(3, 1, "hello world", ""),
}}
res, err := (NonEmptyResultingSegmentValidator{}).Validate(context.Background(), req)
if err != nil {
@@ -144,8 +122,8 @@ func TestNonEmptyResultingSegmentValidator(t *testing.T) {
func TestNoEffectValidator(t *testing.T) {
req := Request{CandidateProposal: []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "hello", "hello", 0.9),
mkCandidate(1, 1, "hello", "hi", 0.9),
mkCandidate(0, 1, "hello", "hello"),
mkCandidate(1, 1, "hello", "hi"),
}}
res, err := (NoEffectValidator{}).Validate(context.Background(), req)
if err != nil {
@@ -166,8 +144,8 @@ func TestValidateProtectedTerms(t *testing.T) {
ProtectedTerms: domain.Vocabulary(entries, nil),
ModuleKey: "grammar",
CandidateProposal: []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "OpenAI", "Open A I", 0.9),
mkCandidate(1, 1, "teh", "the", 0.9),
mkCandidate(0, 1, "OpenAI", "Open A I"),
mkCandidate(1, 1, "teh", "the"),
},
}
res, err := ValidateProtectedTerms(req)
@@ -185,7 +163,6 @@ func TestValidateProtectedTerms(t *testing.T) {
func TestStableReasonCodes(t *testing.T) {
codes := []string{
ReasonApproved,
ReasonLowConfidence,
ReasonMissingOriginalText,
ReasonMissingTargetSegment,
ReasonEmptyResultingText,
@@ -193,7 +170,6 @@ func TestStableReasonCodes(t *testing.T) {
ReasonProtectedGlossaryTerm,
ReasonInvalidTargetSegment,
ReasonEmptyOriginalText,
ReasonInvalidConfidence,
ReasonValidatorMalformed,
ReasonValidatorInputTooLarge,
}

View File

@@ -50,8 +50,8 @@ func TestApplicationChecksCurrentTextWithSelectedProtection(t *testing.T) {
vocabulary := NewProtectedVocabulary(glossary.Vocabulary(protected, active))
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Shield and Aegis"}}}
candidates := []proposals.EnrichedCorrectionProposal{
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "Shield", CorrectedText: "Barrier", Confidence: 1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}},
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "egis", CorrectedText: "rmor", Confidence: 1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}},
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "Shield", CorrectedText: "Barrier"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}},
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "egis", CorrectedText: "rmor"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}},
}
result := proposals.ApplyProposals(transcript, candidates, proposals.ReplacementPolicyRequireUnique, func(before, after string) bool {
return vocabulary.AllowsChange(before, after, false)
@@ -89,8 +89,7 @@ func TestExemptGlossarySpellingKeepsLongestMatchBoundary(t *testing.T) {
terms := glossary.Vocabulary(protected, exclusions)
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: tc.before}}}
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{
TargetSegmentID: 1, OriginalText: tc.original, CorrectedText: tc.replacement, Confidence: 1,
}}
TargetSegmentID: 1, OriginalText: tc.original, CorrectedText: tc.replacement}}
policy := proposals.ReplacementPolicyRequireUnique
decision, err := ValidateProtectedTerms(Request{ModuleKey: tc.module, WorkingTranscript: transcript, ReplacementPolicy: policy,
ProtectedTerms: terms, CandidateProposal: []proposals.EnrichedCorrectionProposal{candidate}})