Share edit location and check protected names in full segments

This commit is contained in:
2026-11-16 04:09:25 +00:00
parent 43ce620122
commit 475557e4de
23 changed files with 488 additions and 269 deletions

View File

@@ -4,6 +4,7 @@ import (
"context"
"strings"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
)
@@ -98,29 +99,25 @@ func (v NoEffectValidator) Validate(_ context.Context, req Request) (Result, err
}
func ValidateProtectedTerms(req Request) (Result, error) {
var vocab ProtectedVocabulary
var vocab glossary.ProtectedVocabulary
if req.GlossaryView != nil {
vocab = req.GlossaryView.Protection
} else {
vocab = NewProtectedVocabulary(req.ProtectedTerms)
vocab = glossary.NewProtectedVocabulary(req.ProtectedTerms)
}
decisions := make([]Decision, 0, len(req.CandidateProposal))
glossaryStage := req.ModuleKey == modulecatalog.KeyGlossary
hasTerms := !vocab.Empty()
if hasTerms {
req.Section = req.sectionLookup()
}
repetitionCleanup := req.ModuleKey == modulecatalog.KeySpokenWord
for _, c := range req.CandidateProposal {
reason := vocab.violationReason(c.OriginalText, c.CorrectedText)
repetitionCleanup := reason != "" && req.ModuleKey == modulecatalog.KeySpokenWord
if vocab.pattern != nil && (glossaryStage || repetitionCleanup) {
req.Section = req.sectionLookup()
var reason string
if hasTerms {
if position, ok := req.Section.Position(c.TargetSegmentID); ok {
segment := &req.Section.segments[position]
preview := req.Preview(c)
if preview.Applicable || (req.IndependentEvidence && preview.SkipReason == proposals.SkipReasonEmptyResultingText) {
if glossaryStage {
// A correction span may sit inside a longer exempt name.
reason = vocab.violationReason(segment.Text, preview.CorrectedSegmentText)
} else if vocab.preservesRepeatedIdentities(segment.Text, preview.CorrectedSegmentText) {
reason = ""
}
if preview.HasResult() {
reason = vocab.Violation(req.Section.segments[position].Text, preview.CorrectedSegmentText, repetitionCleanup)
}
}
}

View File

@@ -11,18 +11,18 @@ import (
// GlossaryView holds the read-only matching data shared by a stage's validators
// and apply-time checks.
type GlossaryView struct {
Protection ProtectedVocabulary
Protection glossary.ProtectedVocabulary
context glossaryContext
}
func PrepareGlossaryView(entries []glossary.Entry, protected []glossary.Term) *GlossaryView {
return &GlossaryView{Protection: NewProtectedVocabulary(protected), context: prepareGlossaryContext(entries)}
return &GlossaryView{Protection: glossary.NewProtectedVocabulary(protected), context: prepareGlossaryContext(entries)}
}
type glossaryContext struct {
entries []glossary.Entry
positions map[glossary.Identity][]int
vocabulary ProtectedVocabulary
vocabulary glossary.ProtectedVocabulary
}
func prepareGlossaryContext(entries []glossary.Entry) glossaryContext {
@@ -31,7 +31,7 @@ func prepareGlossaryContext(entries []glossary.Entry) glossaryContext {
identity := glossary.Identity{Collection: entry.Collection, ID: entry.ID}
positions[identity] = append(positions[identity], i)
}
return glossaryContext{entries: entries, positions: positions, vocabulary: NewProtectedVocabulary(glossary.Vocabulary(entries, nil))}
return glossaryContext{entries: entries, positions: positions, vocabulary: glossary.NewProtectedVocabulary(glossary.Vocabulary(entries, nil))}
}
// validationContext supplies adjacent source segments within the current chunk
@@ -51,12 +51,7 @@ func (view glossaryContext) validationContext(segments []schema.Segment, positio
for _, neighbor := range neighbors {
texts = append(texts, neighbor.Text)
}
mentioned := make(map[glossary.Identity]struct{})
for _, text := range texts {
for _, occurrence := range view.vocabulary.occurrences(text) {
mentioned[occurrence.identity] = struct{}{}
}
}
mentioned := view.vocabulary.MentionedIdentities(texts...)
var matched []int
for identity := range mentioned {
matched = append(matched, view.positions[identity]...)

View File

@@ -1,167 +0,0 @@
package validators
import (
"fmt"
"regexp"
"sort"
"strings"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
)
type protectedTermDef struct {
identities []glossary.Identity
canonical string
}
type protectedOccurrence struct {
text string
identity glossary.Identity
canonical string
}
// ProtectedVocabulary is a deterministic glossary-derived protected term set.
type ProtectedVocabulary struct {
termsByFolded map[string]protectedTermDef
pattern *regexp.Regexp
}
// NewProtectedVocabulary prepares the independently selected terms for matching.
func NewProtectedVocabulary(terms []glossary.Term) ProtectedVocabulary {
termsByFolded := make(map[string]protectedTermDef, len(terms))
alternatives := make([]string, 0, len(terms))
for _, term := range terms {
if !term.Exempt {
termsByFolded[strings.ToLower(term.Spelling)] = protectedTermDef{identities: term.Identities, canonical: term.Spelling}
}
alternatives = append(alternatives, regexp.QuoteMeta(term.Spelling))
}
if len(termsByFolded) == 0 {
return ProtectedVocabulary{}
}
sort.SliceStable(alternatives, func(i, j int) bool { return len(alternatives[i]) > len(alternatives[j]) })
pattern := regexp.MustCompile(`(?i)\b(?:` + strings.Join(alternatives, "|") + `)\b`)
return ProtectedVocabulary{termsByFolded: termsByFolded, pattern: pattern}
}
// AllowsChange checks a proposed segment against the effective protection view.
// Spoken-word cleanup may reduce repeated mentions while preserving each identity.
func (v ProtectedVocabulary) AllowsChange(before, after string, allowRepetitionCleanup bool) bool {
if v.violationReason(before, after) == "" {
return true
}
return allowRepetitionCleanup && v.preservesRepeatedIdentities(before, after)
}
func (v ProtectedVocabulary) violationReason(before, after string) string {
beforeByID := v.occurrencesByIdentity(before)
afterByID := v.occurrencesByIdentity(after)
if reason := validateIdentityPreservation(beforeByID, afterByID); reason != "" {
return reason
}
if reason := validateCapitalizationTransitions(beforeByID, afterByID); reason != "" {
return reason
}
return ""
}
func (v ProtectedVocabulary) occurrencesByIdentity(text string) map[glossary.Identity][]protectedOccurrence {
byID := make(map[glossary.Identity][]protectedOccurrence)
for _, o := range v.occurrences(text) {
byID[o.identity] = append(byID[o.identity], o)
}
return byID
}
func (v ProtectedVocabulary) occurrences(text string) []protectedOccurrence {
if v.pattern == nil {
return nil
}
matches := v.pattern.FindAllStringIndex(text, -1)
if len(matches) == 0 {
return nil
}
out := make([]protectedOccurrence, 0, len(matches))
for _, idx := range matches {
matched := text[idx[0]:idx[1]]
def, ok := v.termsByFolded[strings.ToLower(matched)]
if !ok {
continue
}
for _, identity := range def.identities {
out = append(out, protectedOccurrence{text: matched, identity: identity, canonical: def.canonical})
}
}
return out
}
func validateIdentityPreservation(beforeByID, afterByID map[glossary.Identity][]protectedOccurrence) string {
var selected string
found := false
for identity, beforeItems := range beforeByID {
afterItems := afterByID[identity]
if len(afterItems) >= len(beforeItems) {
continue
}
// Protect identity counts. Within an identity that lost a mention,
// use spelling counts to identify which alias was affected.
remaining := make(map[string]int)
for _, item := range afterItems {
remaining[item.canonical]++
}
for _, item := range beforeItems {
spelling := item.canonical
if remaining[spelling] > 0 {
remaining[spelling]--
continue
}
if !found || spelling < selected {
selected, found = spelling, true
}
}
}
return protectedSpellingMessage(selected, found)
}
func validateCapitalizationTransitions(beforeByID, afterByID map[glossary.Identity][]protectedOccurrence) string {
var selected string
found := false
for identity, afterItems := range afterByID {
beforeItems := beforeByID[identity]
for i, afterItem := range afterItems {
if i >= len(beforeItems) || afterItem.text == beforeItems[i].text || afterItem.text == afterItem.canonical {
continue
}
if !found || afterItem.canonical < selected {
selected, found = afterItem.canonical, true
}
}
}
return protectedSpellingMessage(selected, found)
}
func protectedSpellingMessage(spelling string, found bool) string {
if !found {
return ""
}
return fmt.Sprintf("proposal may alter protected glossary spelling %q", spelling)
}
// preservesRepeatedIdentities permits a spoken-word proposal to reduce repeated
// mentions, but not remove an identity or replace its surviving spelling.
// Editorial and meaning validators still decide whether the cleanup is warranted.
func (v ProtectedVocabulary) preservesRepeatedIdentities(before, after string) bool {
original := v.occurrencesByIdentity(before)
result := v.occurrencesByIdentity(after)
reduced := false
for identity, occurrences := range original {
remaining := result[identity]
if len(remaining) == 0 {
return false
}
if len(remaining) < len(occurrences) {
reduced = true
}
}
return reduced && validateCapitalizationTransitions(original, result) == ""
}

View File

@@ -30,9 +30,11 @@ func TestRecognizedTermsDeduplicateAndIncludePlurals(t *testing.T) {
func TestValidateProtectedTermsAppliesToNonGlossaryModule(t *testing.T) {
req := Request{
ModuleKey: "grammar",
Glossary: []domain.Entry{{ID: "jesters", Name: "Jesters", Category: "faction"}},
ProtectedTerms: domain.Vocabulary([]domain.Entry{{ID: "jesters", Name: "Jesters"}}, nil),
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Jesters"}}},
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
ModuleKey: "grammar",
Glossary: []domain.Entry{{ID: "jesters", Name: "Jesters", Category: "faction"}},
ProtectedTerms: domain.Vocabulary([]domain.Entry{{ID: "jesters", Name: "Jesters"}}, nil),
CandidateProposal: []proposals.EnrichedCorrectionProposal{
{
CorrectionProposal: proposals.CorrectionProposal{
@@ -130,3 +132,32 @@ func TestProtectedNameFindingIdentifiesRemovedAlias(t *testing.T) {
t.Fatalf("finding did not identify the removed alias: %+v", decision)
}
}
func TestProtectionRequiresAUsableFullSegmentPreview(t *testing.T) {
for _, tc := range []struct {
name, source, span, replacement string
target int
rejected bool
}{
{"partial name change", "Cast Mage Armor.", "Mage", "Magus", 1, true},
{"ambiguous span", "Mage Armor and Mage Armor", "Mage Armor", "ward", 1, false},
{"missing span", "Jesters arrived", "Mage Armor", "ward", 1, false},
{"missing target", "Mage Armor", "Mage Armor", "ward", 9, false},
{"empty result", "Mage Armor", "Mage Armor", "", 1, true},
} {
t.Run(tc.name, func(t *testing.T) {
req := Request{
ModuleKey: "grammar", ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
ProtectedTerms: domain.Vocabulary([]domain.Entry{{ID: "mage", Name: "Mage Armor"}}, nil),
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: tc.source}}},
CandidateProposal: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{
TargetSegmentID: tc.target, OriginalText: tc.span, CorrectedText: tc.replacement,
}}},
}
result, err := ValidateProtectedTerms(req)
if err != nil || len(result.Decisions) != 1 || result.Decisions[0].Approved == tc.rejected {
t.Fatalf("protection decision = %+v (%v)", result, err)
}
})
}
}

View File

@@ -140,9 +140,11 @@ func TestNoEffectValidator(t *testing.T) {
func TestValidateProtectedTerms(t *testing.T) {
entries := []domain.Entry{{ID: "openai", Name: "OpenAI", Aliases: []string{"Open AI"}, Category: "brand", Description: "brand"}}
req := Request{
Glossary: entries,
ProtectedTerms: domain.Vocabulary(entries, nil),
ModuleKey: "grammar",
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "OpenAI teh"}}},
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
Glossary: entries,
ProtectedTerms: domain.Vocabulary(entries, nil),
ModuleKey: "grammar",
CandidateProposal: []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "OpenAI", "Open A I"),
mkCandidate(1, 1, "teh", "the"),

View File

@@ -24,7 +24,11 @@ func TestIndependentProtectionForSelectedGlossary(t *testing.T) {
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
result, err := ValidateProtectedTerms(Request{ProtectedTerms: tc.terms, CandidateProposal: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{OriginalText: tc.before, CorrectedText: tc.after}}}})
result, err := ValidateProtectedTerms(Request{
ProtectedTerms: tc.terms, ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: tc.before}}},
CandidateProposal: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: tc.before, CorrectedText: tc.after}}},
})
if err != nil || len(result.Decisions) != 1 || result.Decisions[0].Approved != tc.approved {
t.Fatalf("result=%+v err=%v", result, err)
}
@@ -32,22 +36,10 @@ func TestIndependentProtectionForSelectedGlossary(t *testing.T) {
}
}
func TestSharedAliasRetainsAllIdentities(t *testing.T) {
entries := []glossary.Entry{
{Collection: "a", ID: "first", Name: "First", Aliases: []string{"Shared"}},
{Collection: "b", ID: "second", Name: "Second", Aliases: []string{"Shared"}},
}
vocabulary := NewProtectedVocabulary(glossary.Vocabulary(entries, nil))
identities := vocabulary.occurrencesByIdentity("Shared")
if len(identities) != 2 || len(identities[glossary.Identity{Collection: "a", ID: "first"}]) != 1 || len(identities[glossary.Identity{Collection: "b", ID: "second"}]) != 1 {
t.Fatalf("identities=%v", identities)
}
}
func TestApplicationChecksCurrentTextWithSelectedProtection(t *testing.T) {
active := []glossary.Entry{{Collection: "spells", ID: "spell", Name: "Shield"}}
protected := append(append([]glossary.Entry(nil), active...), glossary.Entry{Collection: "items", ID: "item", Name: "Shield", Aliases: []string{"Aegis"}})
vocabulary := NewProtectedVocabulary(glossary.Vocabulary(protected, active))
vocabulary := glossary.NewProtectedVocabulary(glossary.Vocabulary(protected, active))
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Shield and Aegis"}}}
candidates := []proposals.EnrichedCorrectionProposal{
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "Shield", CorrectedText: "Barrier"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}},
@@ -61,6 +53,32 @@ func TestApplicationChecksCurrentTextWithSelectedProtection(t *testing.T) {
}
}
func TestApplicationRechecksProtectionAfterEarlierEdit(t *testing.T) {
active := []glossary.Entry{{ID: "greater", Name: "Greater Mage Armor"}}
terms := glossary.Vocabulary(append(active, glossary.Entry{ID: "short", Name: "Mage Armor"}), active)
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Cast Greater Mage Armor."}}}
candidates := []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "Greater ", ""),
mkCandidate(1, 1, "Mage Armor", "Magus Armor"),
}
policy := proposals.ReplacementPolicyRequireUnique
validation, err := ValidateProtectedTerms(Request{
ModuleKey: "glossary", WorkingTranscript: transcript, ReplacementPolicy: policy,
ProtectedTerms: terms, CandidateProposal: candidates,
})
if err != nil || len(validation.Decisions) != 2 || !validation.Decisions[0].Approved || !validation.Decisions[1].Approved {
t.Fatalf("initial decisions = %+v (%v)", validation, err)
}
vocabulary := glossary.NewProtectedVocabulary(terms)
result := proposals.ApplyProposals(transcript, candidates, policy, func(before, after string) bool {
return vocabulary.AllowsChange(before, after, false)
})
if result.Transcript.Segments[0].Text != "Cast Mage Armor." || len(result.Applied) != 1 ||
len(result.Skipped) != 1 || result.Skipped[0].ProposalIndex != 1 || result.Skipped[0].SkipReason != proposals.SkipReasonProtectedTerm {
t.Fatalf("stale proposal outcome: %+v", result)
}
}
func TestExemptGlossarySpellingKeepsLongestMatchBoundary(t *testing.T) {
active := []glossary.Entry{
{ID: "greater-mage", Name: "Greater Mage Armor"},
@@ -96,7 +114,7 @@ func TestExemptGlossarySpellingKeepsLongestMatchBoundary(t *testing.T) {
if err != nil || len(decision.Decisions) != 1 || decision.Decisions[0].Approved != tc.allowed {
t.Fatalf("decision=%+v err=%v", decision, err)
}
vocabulary := NewProtectedVocabulary(terms)
vocabulary := glossary.NewProtectedVocabulary(terms)
result := proposals.ApplyProposals(transcript, []proposals.EnrichedCorrectionProposal{candidate}, policy, func(before, after string) bool {
return vocabulary.AllowsChange(before, after, false)
})