Keep match boundaries for exempt glossary names
This commit is contained in:
@@ -49,10 +49,11 @@ nonempty, unique `id` identifies a record in evidence. Each record may override
|
|||||||
Explicit empty lists and null list selections clear a selection.
|
Explicit empty lists and null list selections clear a selection.
|
||||||
|
|
||||||
Each entry receives its selected context and independent protection. Glossary
|
Each entry receives its selected context and independent protection. Glossary
|
||||||
correction exempts active recognized spellings from protection for that entry;
|
correction exempts active recognized spellings from protection for that entry.
|
||||||
other protected spellings remain protected. A glossary correction must have at
|
A longer exempt name still takes precedence over a shorter protected spelling
|
||||||
least one selected entry at processing preflight. Empty definitions and empty
|
inside it. A glossary correction must have at least one selected entry at
|
||||||
protection remain valid configuration.
|
processing preflight. Empty definitions and empty protection remain valid
|
||||||
|
configuration.
|
||||||
|
|
||||||
## Settings
|
## Settings
|
||||||
|
|
||||||
|
|||||||
@@ -416,6 +416,8 @@ prepared sibling file, and failed delivery cleans up that file.
|
|||||||
|
|
||||||
## R12 — P2: Preserve Longest-Match Context for Glossary Exemptions
|
## R12 — P2: Preserve Longest-Match Context for Glossary Exemptions
|
||||||
|
|
||||||
|
**Status:** Completed. Active spellings remain longest-match boundaries without contributing protected identities. Tests cover full-name and span edits, standalone and longer protected terms, stage isolation, and apply-time agreement.
|
||||||
|
|
||||||
**Category**
|
**Category**
|
||||||
|
|
||||||
The new Stage 11 exemption changes how the existing matcher identifies
|
The new Stage 11 exemption changes how the existing matcher identifies
|
||||||
|
|||||||
@@ -8,19 +8,25 @@ type Identity struct {
|
|||||||
ID string
|
ID string
|
||||||
}
|
}
|
||||||
|
|
||||||
// Term retains every entry that contributes a recognized spelling.
|
// Term is a recognized spelling and its protected identities, if any.
|
||||||
type Term struct {
|
type Term struct {
|
||||||
Spelling string
|
Spelling string
|
||||||
Identities []Identity
|
Identities []Identity
|
||||||
|
Exempt bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// Vocabulary derives recognized spellings in selection order. Excluded entries
|
// Vocabulary derives recognized spellings in selection order. Active spellings
|
||||||
// remove matching spellings regardless of which protected entries supplied them.
|
// are exempt from protection but still participate in longest matching.
|
||||||
func Vocabulary(entries, excluded []Entry) []Term {
|
func Vocabulary(entries, excluded []Entry) []Term {
|
||||||
exclusions := make(map[string]struct{})
|
exclusions := make(map[string]struct{})
|
||||||
|
var exempt []Term
|
||||||
for _, entry := range excluded {
|
for _, entry := range excluded {
|
||||||
for _, spelling := range spellings(entry) {
|
for _, spelling := range spellings(entry) {
|
||||||
exclusions[strings.ToLower(spelling)] = struct{}{}
|
key := strings.ToLower(spelling)
|
||||||
|
if _, exists := exclusions[key]; !exists {
|
||||||
|
exclusions[key] = struct{}{}
|
||||||
|
exempt = append(exempt, Term{Spelling: spelling, Exempt: true})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
indices := make(map[string]int)
|
indices := make(map[string]int)
|
||||||
@@ -51,7 +57,7 @@ func Vocabulary(entries, excluded []Entry) []Term {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return terms
|
return append(terms, exempt...)
|
||||||
}
|
}
|
||||||
|
|
||||||
func spellings(entry Entry) []string {
|
func spellings(entry Entry) []string {
|
||||||
|
|||||||
@@ -17,12 +17,17 @@ func TestVocabularyKeepsContributorsAndExemptsOnlyActiveSpellings(t *testing.T)
|
|||||||
t.Fatalf("shared spelling contributors: %+v", full[0])
|
t.Fatalf("shared spelling contributors: %+v", full[0])
|
||||||
}
|
}
|
||||||
reduced := Vocabulary(protected, active)
|
reduced := Vocabulary(protected, active)
|
||||||
var got []string
|
var remaining, exempt []string
|
||||||
for _, term := range reduced {
|
for _, term := range reduced {
|
||||||
got = append(got, term.Spelling)
|
if term.Exempt {
|
||||||
|
exempt = append(exempt, term.Spelling)
|
||||||
|
} else {
|
||||||
|
remaining = append(remaining, term.Spelling)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if !reflect.DeepEqual(got, []string{"Aegis", "Aegiss", "Aegides"}) {
|
if !reflect.DeepEqual(remaining, []string{"Aegis", "Aegiss", "Aegides"}) ||
|
||||||
t.Fatalf("remaining spellings: %v", got)
|
!reflect.DeepEqual(exempt, []string{"Shield", "Shields", "Ward", "Wards"}) {
|
||||||
|
t.Fatalf("protected: %v; exempt: %v", remaining, exempt)
|
||||||
}
|
}
|
||||||
if !reflect.DeepEqual(full, Vocabulary(protected, nil)) || protected[0].Aliases[0] != "Ward" {
|
if !reflect.DeepEqual(full, Vocabulary(protected, nil)) || protected[0].Aliases[0] != "Ward" {
|
||||||
t.Fatal("deriving an exemption changed another view or its inputs")
|
t.Fatal("deriving an exemption changed another view or its inputs")
|
||||||
|
|||||||
@@ -50,7 +50,9 @@ func ExecuteModuleProposal(ctx context.Context, req ModuleProposalRequest) (cont
|
|||||||
|
|
||||||
protected := make([]string, 0, len(req.ProposalRequest.ProtectedTerms))
|
protected := make([]string, 0, len(req.ProposalRequest.ProtectedTerms))
|
||||||
for _, term := range req.ProposalRequest.ProtectedTerms {
|
for _, term := range req.ProposalRequest.ProtectedTerms {
|
||||||
protected = append(protected, term.Spelling)
|
if !term.Exempt {
|
||||||
|
protected = append(protected, term.Spelling)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
payload, err := json.Marshal(protected)
|
payload, err := json.Marshal(protected)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -40,7 +40,7 @@ func TestExecuteModuleProposalBuildsMessagesFromSectionAndDescription(t *testing
|
|||||||
Settings: cfg,
|
Settings: cfg,
|
||||||
WorkingTranscript: transcript,
|
WorkingTranscript: transcript,
|
||||||
Glossary: glossary,
|
Glossary: glossary,
|
||||||
ProtectedTerms: []domain.Term{{Spelling: "Separate protection"}},
|
ProtectedTerms: []domain.Term{{Spelling: "Separate protection"}, {Spelling: "Exempt boundary", Exempt: true}},
|
||||||
Section: §ion,
|
Section: §ion,
|
||||||
},
|
},
|
||||||
RunSpec: contracts.ModuleRunSpec{
|
RunSpec: contracts.ModuleRunSpec{
|
||||||
|
|||||||
@@ -139,17 +139,24 @@ func (v NoEffectValidator) Validate(_ context.Context, req Request) (Result, err
|
|||||||
func ValidateProtectedTerms(req Request) (Result, error) {
|
func ValidateProtectedTerms(req Request) (Result, error) {
|
||||||
vocab := NewProtectedVocabulary(req.ProtectedTerms)
|
vocab := NewProtectedVocabulary(req.ProtectedTerms)
|
||||||
decisions := make([]Decision, 0, len(req.CandidateProposal))
|
decisions := make([]Decision, 0, len(req.CandidateProposal))
|
||||||
|
glossaryStage := req.ModuleKey == modulecatalog.KeyGlossary
|
||||||
for _, c := range req.CandidateProposal {
|
for _, c := range req.CandidateProposal {
|
||||||
reason := vocab.violationReason(c.OriginalText, c.CorrectedText)
|
reason := vocab.violationReason(c.OriginalText, c.CorrectedText)
|
||||||
if reason != "" && req.ModuleKey == modulecatalog.KeySpokenWord && req.WorkingTranscript != nil {
|
repetitionCleanup := reason != "" && req.ModuleKey == modulecatalog.KeySpokenWord
|
||||||
|
if vocab.pattern != nil && (glossaryStage || repetitionCleanup) && req.WorkingTranscript != nil {
|
||||||
for i := range req.WorkingTranscript.Segments {
|
for i := range req.WorkingTranscript.Segments {
|
||||||
segment := &req.WorkingTranscript.Segments[i]
|
segment := &req.WorkingTranscript.Segments[i]
|
||||||
if segment.ID != c.TargetSegmentID {
|
if segment.ID != c.TargetSegmentID {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
preview := proposals.PreviewProposalForSegment(segment, c.CorrectionProposal, req.ReplacementPolicy)
|
preview := proposals.PreviewProposalForSegment(segment, c.CorrectionProposal, req.ReplacementPolicy)
|
||||||
if preview.Applicable && vocab.preservesRepeatedIdentities(segment.Text, preview.CorrectedSegmentText) {
|
if preview.Applicable {
|
||||||
reason = ""
|
if glossaryStage {
|
||||||
|
// A correction span may sit inside a longer exempt name.
|
||||||
|
reason = vocab.violationReason(segment.Text, preview.CorrectedSegmentText)
|
||||||
|
} else if vocab.preservesRepeatedIdentities(segment.Text, preview.CorrectedSegmentText) {
|
||||||
|
reason = ""
|
||||||
|
}
|
||||||
}
|
}
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -30,13 +30,15 @@ func NewProtectedVocabulary(terms []glossary.Term) ProtectedVocabulary {
|
|||||||
termsByFolded := make(map[string]protectedTermDef, len(terms))
|
termsByFolded := make(map[string]protectedTermDef, len(terms))
|
||||||
alternatives := make([]string, 0, len(terms))
|
alternatives := make([]string, 0, len(terms))
|
||||||
for _, term := range terms {
|
for _, term := range terms {
|
||||||
termsByFolded[strings.ToLower(term.Spelling)] = protectedTermDef{identities: term.Identities, canonical: term.Spelling}
|
if !term.Exempt {
|
||||||
|
termsByFolded[strings.ToLower(term.Spelling)] = protectedTermDef{identities: term.Identities, canonical: term.Spelling}
|
||||||
|
}
|
||||||
alternatives = append(alternatives, regexp.QuoteMeta(term.Spelling))
|
alternatives = append(alternatives, regexp.QuoteMeta(term.Spelling))
|
||||||
}
|
}
|
||||||
sort.SliceStable(alternatives, func(i, j int) bool { return len(alternatives[i]) > len(alternatives[j]) })
|
if len(termsByFolded) == 0 {
|
||||||
if len(alternatives) == 0 {
|
return ProtectedVocabulary{}
|
||||||
return ProtectedVocabulary{termsByFolded: termsByFolded}
|
|
||||||
}
|
}
|
||||||
|
sort.SliceStable(alternatives, func(i, j int) bool { return len(alternatives[i]) > len(alternatives[j]) })
|
||||||
pattern := regexp.MustCompile(`(?i)\b(?:` + strings.Join(alternatives, "|") + `)\b`)
|
pattern := regexp.MustCompile(`(?i)\b(?:` + strings.Join(alternatives, "|") + `)\b`)
|
||||||
return ProtectedVocabulary{termsByFolded: termsByFolded, pattern: pattern}
|
return ProtectedVocabulary{termsByFolded: termsByFolded, pattern: pattern}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -61,6 +61,53 @@ func TestApplicationChecksCurrentTextWithSelectedProtection(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExemptGlossarySpellingKeepsLongestMatchBoundary(t *testing.T) {
|
||||||
|
active := []glossary.Entry{
|
||||||
|
{ID: "greater-mage", Name: "Greater Mage Armor"},
|
||||||
|
{ID: "greater-magus", Name: "Greater Magus Armor"},
|
||||||
|
}
|
||||||
|
protected := []glossary.Entry{
|
||||||
|
active[0], active[1],
|
||||||
|
{ID: "short", Name: "Mage Armor"},
|
||||||
|
{ID: "long", Name: "Superior Greater Mage Armor"},
|
||||||
|
}
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name, module, before, original, replacement string
|
||||||
|
exempt, allowed bool
|
||||||
|
}{
|
||||||
|
{"full active name", "glossary", "Cast Greater Mage Armor.", "Greater Mage Armor", "Greater Magus Armor", true, true},
|
||||||
|
{"span within active name", "glossary", "Cast Greater Mage Armor.", "Mage Armor", "Magus Armor", true, true},
|
||||||
|
{"standalone protected name", "glossary", "Cast Mage Armor.", "Mage Armor", "Magus Armor", true, false},
|
||||||
|
{"longer protected name", "glossary", "Cast Superior Greater Mage Armor.", "Greater Mage Armor", "Greater Magus Armor", true, false},
|
||||||
|
{"same name in a grammar pass", "grammar", "Cast Greater Mage Armor.", "Greater Mage Armor", "Greater Magus Armor", false, false},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
var exclusions []glossary.Entry
|
||||||
|
if tc.exempt {
|
||||||
|
exclusions = active
|
||||||
|
}
|
||||||
|
terms := glossary.Vocabulary(protected, exclusions)
|
||||||
|
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: tc.before}}}
|
||||||
|
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{
|
||||||
|
TargetSegmentID: 1, OriginalText: tc.original, CorrectedText: tc.replacement, Confidence: 1,
|
||||||
|
}}
|
||||||
|
policy := proposals.ReplacementPolicyRequireUnique
|
||||||
|
decision, err := ValidateProtectedTerms(Request{ModuleKey: tc.module, WorkingTranscript: transcript, ReplacementPolicy: policy,
|
||||||
|
ProtectedTerms: terms, CandidateProposal: []proposals.EnrichedCorrectionProposal{candidate}})
|
||||||
|
if err != nil || len(decision.Decisions) != 1 || decision.Decisions[0].Approved != tc.allowed {
|
||||||
|
t.Fatalf("decision=%+v err=%v", decision, err)
|
||||||
|
}
|
||||||
|
vocabulary := NewProtectedVocabulary(terms)
|
||||||
|
result := proposals.ApplyProposals(transcript, []proposals.EnrichedCorrectionProposal{candidate}, policy, func(before, after string) bool {
|
||||||
|
return vocabulary.AllowsChange(before, after, false)
|
||||||
|
})
|
||||||
|
if (len(result.Applied) == 1) != tc.allowed {
|
||||||
|
t.Fatalf("apply-time protection disagrees with validation: %+v", result)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestValidationContextIncludesEachSharedDefinition(t *testing.T) {
|
func TestValidationContextIncludesEachSharedDefinition(t *testing.T) {
|
||||||
entries := []glossary.Entry{
|
entries := []glossary.Entry{
|
||||||
{Collection: "spells", ID: "shield", Name: "Shield", Category: "spell"},
|
{Collection: "spells", ID: "shield", Name: "Shield", Category: "spell"},
|
||||||
|
|||||||
Reference in New Issue
Block a user