Share edit location and check protected names in full segments
This commit is contained in:
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||
)
|
||||
@@ -98,29 +99,25 @@ func (v NoEffectValidator) Validate(_ context.Context, req Request) (Result, err
|
||||
}
|
||||
|
||||
func ValidateProtectedTerms(req Request) (Result, error) {
|
||||
var vocab ProtectedVocabulary
|
||||
var vocab glossary.ProtectedVocabulary
|
||||
if req.GlossaryView != nil {
|
||||
vocab = req.GlossaryView.Protection
|
||||
} else {
|
||||
vocab = NewProtectedVocabulary(req.ProtectedTerms)
|
||||
vocab = glossary.NewProtectedVocabulary(req.ProtectedTerms)
|
||||
}
|
||||
decisions := make([]Decision, 0, len(req.CandidateProposal))
|
||||
glossaryStage := req.ModuleKey == modulecatalog.KeyGlossary
|
||||
hasTerms := !vocab.Empty()
|
||||
if hasTerms {
|
||||
req.Section = req.sectionLookup()
|
||||
}
|
||||
repetitionCleanup := req.ModuleKey == modulecatalog.KeySpokenWord
|
||||
for _, c := range req.CandidateProposal {
|
||||
reason := vocab.violationReason(c.OriginalText, c.CorrectedText)
|
||||
repetitionCleanup := reason != "" && req.ModuleKey == modulecatalog.KeySpokenWord
|
||||
if vocab.pattern != nil && (glossaryStage || repetitionCleanup) {
|
||||
req.Section = req.sectionLookup()
|
||||
var reason string
|
||||
if hasTerms {
|
||||
if position, ok := req.Section.Position(c.TargetSegmentID); ok {
|
||||
segment := &req.Section.segments[position]
|
||||
preview := req.Preview(c)
|
||||
if preview.Applicable || (req.IndependentEvidence && preview.SkipReason == proposals.SkipReasonEmptyResultingText) {
|
||||
if glossaryStage {
|
||||
// A correction span may sit inside a longer exempt name.
|
||||
reason = vocab.violationReason(segment.Text, preview.CorrectedSegmentText)
|
||||
} else if vocab.preservesRepeatedIdentities(segment.Text, preview.CorrectedSegmentText) {
|
||||
reason = ""
|
||||
}
|
||||
if preview.HasResult() {
|
||||
reason = vocab.Violation(req.Section.segments[position].Text, preview.CorrectedSegmentText, repetitionCleanup)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,18 +11,18 @@ import (
|
||||
// GlossaryView holds the read-only matching data shared by a stage's validators
|
||||
// and apply-time checks.
|
||||
type GlossaryView struct {
|
||||
Protection ProtectedVocabulary
|
||||
Protection glossary.ProtectedVocabulary
|
||||
context glossaryContext
|
||||
}
|
||||
|
||||
func PrepareGlossaryView(entries []glossary.Entry, protected []glossary.Term) *GlossaryView {
|
||||
return &GlossaryView{Protection: NewProtectedVocabulary(protected), context: prepareGlossaryContext(entries)}
|
||||
return &GlossaryView{Protection: glossary.NewProtectedVocabulary(protected), context: prepareGlossaryContext(entries)}
|
||||
}
|
||||
|
||||
type glossaryContext struct {
|
||||
entries []glossary.Entry
|
||||
positions map[glossary.Identity][]int
|
||||
vocabulary ProtectedVocabulary
|
||||
vocabulary glossary.ProtectedVocabulary
|
||||
}
|
||||
|
||||
func prepareGlossaryContext(entries []glossary.Entry) glossaryContext {
|
||||
@@ -31,7 +31,7 @@ func prepareGlossaryContext(entries []glossary.Entry) glossaryContext {
|
||||
identity := glossary.Identity{Collection: entry.Collection, ID: entry.ID}
|
||||
positions[identity] = append(positions[identity], i)
|
||||
}
|
||||
return glossaryContext{entries: entries, positions: positions, vocabulary: NewProtectedVocabulary(glossary.Vocabulary(entries, nil))}
|
||||
return glossaryContext{entries: entries, positions: positions, vocabulary: glossary.NewProtectedVocabulary(glossary.Vocabulary(entries, nil))}
|
||||
}
|
||||
|
||||
// validationContext supplies adjacent source segments within the current chunk
|
||||
@@ -51,12 +51,7 @@ func (view glossaryContext) validationContext(segments []schema.Segment, positio
|
||||
for _, neighbor := range neighbors {
|
||||
texts = append(texts, neighbor.Text)
|
||||
}
|
||||
mentioned := make(map[glossary.Identity]struct{})
|
||||
for _, text := range texts {
|
||||
for _, occurrence := range view.vocabulary.occurrences(text) {
|
||||
mentioned[occurrence.identity] = struct{}{}
|
||||
}
|
||||
}
|
||||
mentioned := view.vocabulary.MentionedIdentities(texts...)
|
||||
var matched []int
|
||||
for identity := range mentioned {
|
||||
matched = append(matched, view.positions[identity]...)
|
||||
|
||||
@@ -1,167 +0,0 @@
|
||||
package validators
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
|
||||
)
|
||||
|
||||
type protectedTermDef struct {
|
||||
identities []glossary.Identity
|
||||
canonical string
|
||||
}
|
||||
|
||||
type protectedOccurrence struct {
|
||||
text string
|
||||
identity glossary.Identity
|
||||
canonical string
|
||||
}
|
||||
|
||||
// ProtectedVocabulary is a deterministic glossary-derived protected term set.
|
||||
type ProtectedVocabulary struct {
|
||||
termsByFolded map[string]protectedTermDef
|
||||
pattern *regexp.Regexp
|
||||
}
|
||||
|
||||
// NewProtectedVocabulary prepares the independently selected terms for matching.
|
||||
func NewProtectedVocabulary(terms []glossary.Term) ProtectedVocabulary {
|
||||
termsByFolded := make(map[string]protectedTermDef, len(terms))
|
||||
alternatives := make([]string, 0, len(terms))
|
||||
for _, term := range terms {
|
||||
if !term.Exempt {
|
||||
termsByFolded[strings.ToLower(term.Spelling)] = protectedTermDef{identities: term.Identities, canonical: term.Spelling}
|
||||
}
|
||||
alternatives = append(alternatives, regexp.QuoteMeta(term.Spelling))
|
||||
}
|
||||
if len(termsByFolded) == 0 {
|
||||
return ProtectedVocabulary{}
|
||||
}
|
||||
sort.SliceStable(alternatives, func(i, j int) bool { return len(alternatives[i]) > len(alternatives[j]) })
|
||||
pattern := regexp.MustCompile(`(?i)\b(?:` + strings.Join(alternatives, "|") + `)\b`)
|
||||
return ProtectedVocabulary{termsByFolded: termsByFolded, pattern: pattern}
|
||||
}
|
||||
|
||||
// AllowsChange checks a proposed segment against the effective protection view.
|
||||
// Spoken-word cleanup may reduce repeated mentions while preserving each identity.
|
||||
func (v ProtectedVocabulary) AllowsChange(before, after string, allowRepetitionCleanup bool) bool {
|
||||
if v.violationReason(before, after) == "" {
|
||||
return true
|
||||
}
|
||||
return allowRepetitionCleanup && v.preservesRepeatedIdentities(before, after)
|
||||
}
|
||||
|
||||
func (v ProtectedVocabulary) violationReason(before, after string) string {
|
||||
beforeByID := v.occurrencesByIdentity(before)
|
||||
afterByID := v.occurrencesByIdentity(after)
|
||||
if reason := validateIdentityPreservation(beforeByID, afterByID); reason != "" {
|
||||
return reason
|
||||
}
|
||||
if reason := validateCapitalizationTransitions(beforeByID, afterByID); reason != "" {
|
||||
return reason
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (v ProtectedVocabulary) occurrencesByIdentity(text string) map[glossary.Identity][]protectedOccurrence {
|
||||
byID := make(map[glossary.Identity][]protectedOccurrence)
|
||||
for _, o := range v.occurrences(text) {
|
||||
byID[o.identity] = append(byID[o.identity], o)
|
||||
}
|
||||
return byID
|
||||
}
|
||||
|
||||
func (v ProtectedVocabulary) occurrences(text string) []protectedOccurrence {
|
||||
if v.pattern == nil {
|
||||
return nil
|
||||
}
|
||||
matches := v.pattern.FindAllStringIndex(text, -1)
|
||||
if len(matches) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]protectedOccurrence, 0, len(matches))
|
||||
for _, idx := range matches {
|
||||
matched := text[idx[0]:idx[1]]
|
||||
def, ok := v.termsByFolded[strings.ToLower(matched)]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for _, identity := range def.identities {
|
||||
out = append(out, protectedOccurrence{text: matched, identity: identity, canonical: def.canonical})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func validateIdentityPreservation(beforeByID, afterByID map[glossary.Identity][]protectedOccurrence) string {
|
||||
var selected string
|
||||
found := false
|
||||
for identity, beforeItems := range beforeByID {
|
||||
afterItems := afterByID[identity]
|
||||
if len(afterItems) >= len(beforeItems) {
|
||||
continue
|
||||
}
|
||||
// Protect identity counts. Within an identity that lost a mention,
|
||||
// use spelling counts to identify which alias was affected.
|
||||
remaining := make(map[string]int)
|
||||
for _, item := range afterItems {
|
||||
remaining[item.canonical]++
|
||||
}
|
||||
for _, item := range beforeItems {
|
||||
spelling := item.canonical
|
||||
if remaining[spelling] > 0 {
|
||||
remaining[spelling]--
|
||||
continue
|
||||
}
|
||||
if !found || spelling < selected {
|
||||
selected, found = spelling, true
|
||||
}
|
||||
}
|
||||
}
|
||||
return protectedSpellingMessage(selected, found)
|
||||
}
|
||||
|
||||
func validateCapitalizationTransitions(beforeByID, afterByID map[glossary.Identity][]protectedOccurrence) string {
|
||||
var selected string
|
||||
found := false
|
||||
for identity, afterItems := range afterByID {
|
||||
beforeItems := beforeByID[identity]
|
||||
for i, afterItem := range afterItems {
|
||||
if i >= len(beforeItems) || afterItem.text == beforeItems[i].text || afterItem.text == afterItem.canonical {
|
||||
continue
|
||||
}
|
||||
if !found || afterItem.canonical < selected {
|
||||
selected, found = afterItem.canonical, true
|
||||
}
|
||||
}
|
||||
}
|
||||
return protectedSpellingMessage(selected, found)
|
||||
}
|
||||
|
||||
func protectedSpellingMessage(spelling string, found bool) string {
|
||||
if !found {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("proposal may alter protected glossary spelling %q", spelling)
|
||||
}
|
||||
|
||||
// preservesRepeatedIdentities permits a spoken-word proposal to reduce repeated
|
||||
// mentions, but not remove an identity or replace its surviving spelling.
|
||||
// Editorial and meaning validators still decide whether the cleanup is warranted.
|
||||
func (v ProtectedVocabulary) preservesRepeatedIdentities(before, after string) bool {
|
||||
original := v.occurrencesByIdentity(before)
|
||||
result := v.occurrencesByIdentity(after)
|
||||
reduced := false
|
||||
for identity, occurrences := range original {
|
||||
remaining := result[identity]
|
||||
if len(remaining) == 0 {
|
||||
return false
|
||||
}
|
||||
if len(remaining) < len(occurrences) {
|
||||
reduced = true
|
||||
}
|
||||
}
|
||||
return reduced && validateCapitalizationTransitions(original, result) == ""
|
||||
}
|
||||
@@ -30,9 +30,11 @@ func TestRecognizedTermsDeduplicateAndIncludePlurals(t *testing.T) {
|
||||
|
||||
func TestValidateProtectedTermsAppliesToNonGlossaryModule(t *testing.T) {
|
||||
req := Request{
|
||||
ModuleKey: "grammar",
|
||||
Glossary: []domain.Entry{{ID: "jesters", Name: "Jesters", Category: "faction"}},
|
||||
ProtectedTerms: domain.Vocabulary([]domain.Entry{{ID: "jesters", Name: "Jesters"}}, nil),
|
||||
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Jesters"}}},
|
||||
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
||||
ModuleKey: "grammar",
|
||||
Glossary: []domain.Entry{{ID: "jesters", Name: "Jesters", Category: "faction"}},
|
||||
ProtectedTerms: domain.Vocabulary([]domain.Entry{{ID: "jesters", Name: "Jesters"}}, nil),
|
||||
CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
||||
{
|
||||
CorrectionProposal: proposals.CorrectionProposal{
|
||||
@@ -130,3 +132,32 @@ func TestProtectedNameFindingIdentifiesRemovedAlias(t *testing.T) {
|
||||
t.Fatalf("finding did not identify the removed alias: %+v", decision)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProtectionRequiresAUsableFullSegmentPreview(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, source, span, replacement string
|
||||
target int
|
||||
rejected bool
|
||||
}{
|
||||
{"partial name change", "Cast Mage Armor.", "Mage", "Magus", 1, true},
|
||||
{"ambiguous span", "Mage Armor and Mage Armor", "Mage Armor", "ward", 1, false},
|
||||
{"missing span", "Jesters arrived", "Mage Armor", "ward", 1, false},
|
||||
{"missing target", "Mage Armor", "Mage Armor", "ward", 9, false},
|
||||
{"empty result", "Mage Armor", "Mage Armor", "", 1, true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
req := Request{
|
||||
ModuleKey: "grammar", ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
||||
ProtectedTerms: domain.Vocabulary([]domain.Entry{{ID: "mage", Name: "Mage Armor"}}, nil),
|
||||
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: tc.source}}},
|
||||
CandidateProposal: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{
|
||||
TargetSegmentID: tc.target, OriginalText: tc.span, CorrectedText: tc.replacement,
|
||||
}}},
|
||||
}
|
||||
result, err := ValidateProtectedTerms(req)
|
||||
if err != nil || len(result.Decisions) != 1 || result.Decisions[0].Approved == tc.rejected {
|
||||
t.Fatalf("protection decision = %+v (%v)", result, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -140,9 +140,11 @@ func TestNoEffectValidator(t *testing.T) {
|
||||
func TestValidateProtectedTerms(t *testing.T) {
|
||||
entries := []domain.Entry{{ID: "openai", Name: "OpenAI", Aliases: []string{"Open AI"}, Category: "brand", Description: "brand"}}
|
||||
req := Request{
|
||||
Glossary: entries,
|
||||
ProtectedTerms: domain.Vocabulary(entries, nil),
|
||||
ModuleKey: "grammar",
|
||||
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "OpenAI teh"}}},
|
||||
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
||||
Glossary: entries,
|
||||
ProtectedTerms: domain.Vocabulary(entries, nil),
|
||||
ModuleKey: "grammar",
|
||||
CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
||||
mkCandidate(0, 1, "OpenAI", "Open A I"),
|
||||
mkCandidate(1, 1, "teh", "the"),
|
||||
|
||||
@@ -24,7 +24,11 @@ func TestIndependentProtectionForSelectedGlossary(t *testing.T) {
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
result, err := ValidateProtectedTerms(Request{ProtectedTerms: tc.terms, CandidateProposal: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{OriginalText: tc.before, CorrectedText: tc.after}}}})
|
||||
result, err := ValidateProtectedTerms(Request{
|
||||
ProtectedTerms: tc.terms, ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
||||
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: tc.before}}},
|
||||
CandidateProposal: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: tc.before, CorrectedText: tc.after}}},
|
||||
})
|
||||
if err != nil || len(result.Decisions) != 1 || result.Decisions[0].Approved != tc.approved {
|
||||
t.Fatalf("result=%+v err=%v", result, err)
|
||||
}
|
||||
@@ -32,22 +36,10 @@ func TestIndependentProtectionForSelectedGlossary(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSharedAliasRetainsAllIdentities(t *testing.T) {
|
||||
entries := []glossary.Entry{
|
||||
{Collection: "a", ID: "first", Name: "First", Aliases: []string{"Shared"}},
|
||||
{Collection: "b", ID: "second", Name: "Second", Aliases: []string{"Shared"}},
|
||||
}
|
||||
vocabulary := NewProtectedVocabulary(glossary.Vocabulary(entries, nil))
|
||||
identities := vocabulary.occurrencesByIdentity("Shared")
|
||||
if len(identities) != 2 || len(identities[glossary.Identity{Collection: "a", ID: "first"}]) != 1 || len(identities[glossary.Identity{Collection: "b", ID: "second"}]) != 1 {
|
||||
t.Fatalf("identities=%v", identities)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplicationChecksCurrentTextWithSelectedProtection(t *testing.T) {
|
||||
active := []glossary.Entry{{Collection: "spells", ID: "spell", Name: "Shield"}}
|
||||
protected := append(append([]glossary.Entry(nil), active...), glossary.Entry{Collection: "items", ID: "item", Name: "Shield", Aliases: []string{"Aegis"}})
|
||||
vocabulary := NewProtectedVocabulary(glossary.Vocabulary(protected, active))
|
||||
vocabulary := glossary.NewProtectedVocabulary(glossary.Vocabulary(protected, active))
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Shield and Aegis"}}}
|
||||
candidates := []proposals.EnrichedCorrectionProposal{
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "Shield", CorrectedText: "Barrier"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}},
|
||||
@@ -61,6 +53,32 @@ func TestApplicationChecksCurrentTextWithSelectedProtection(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplicationRechecksProtectionAfterEarlierEdit(t *testing.T) {
|
||||
active := []glossary.Entry{{ID: "greater", Name: "Greater Mage Armor"}}
|
||||
terms := glossary.Vocabulary(append(active, glossary.Entry{ID: "short", Name: "Mage Armor"}), active)
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Cast Greater Mage Armor."}}}
|
||||
candidates := []proposals.EnrichedCorrectionProposal{
|
||||
mkCandidate(0, 1, "Greater ", ""),
|
||||
mkCandidate(1, 1, "Mage Armor", "Magus Armor"),
|
||||
}
|
||||
policy := proposals.ReplacementPolicyRequireUnique
|
||||
validation, err := ValidateProtectedTerms(Request{
|
||||
ModuleKey: "glossary", WorkingTranscript: transcript, ReplacementPolicy: policy,
|
||||
ProtectedTerms: terms, CandidateProposal: candidates,
|
||||
})
|
||||
if err != nil || len(validation.Decisions) != 2 || !validation.Decisions[0].Approved || !validation.Decisions[1].Approved {
|
||||
t.Fatalf("initial decisions = %+v (%v)", validation, err)
|
||||
}
|
||||
vocabulary := glossary.NewProtectedVocabulary(terms)
|
||||
result := proposals.ApplyProposals(transcript, candidates, policy, func(before, after string) bool {
|
||||
return vocabulary.AllowsChange(before, after, false)
|
||||
})
|
||||
if result.Transcript.Segments[0].Text != "Cast Mage Armor." || len(result.Applied) != 1 ||
|
||||
len(result.Skipped) != 1 || result.Skipped[0].ProposalIndex != 1 || result.Skipped[0].SkipReason != proposals.SkipReasonProtectedTerm {
|
||||
t.Fatalf("stale proposal outcome: %+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExemptGlossarySpellingKeepsLongestMatchBoundary(t *testing.T) {
|
||||
active := []glossary.Entry{
|
||||
{ID: "greater-mage", Name: "Greater Mage Armor"},
|
||||
@@ -96,7 +114,7 @@ func TestExemptGlossarySpellingKeepsLongestMatchBoundary(t *testing.T) {
|
||||
if err != nil || len(decision.Decisions) != 1 || decision.Decisions[0].Approved != tc.allowed {
|
||||
t.Fatalf("decision=%+v err=%v", decision, err)
|
||||
}
|
||||
vocabulary := NewProtectedVocabulary(terms)
|
||||
vocabulary := glossary.NewProtectedVocabulary(terms)
|
||||
result := proposals.ApplyProposals(transcript, []proposals.EnrichedCorrectionProposal{candidate}, policy, func(before, after string) bool {
|
||||
return vocabulary.AllowsChange(before, after, false)
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user