Share edit location and check protected names in full segments

This commit is contained in:
2026-11-16 04:09:25 +00:00
parent 43ce620122
commit 475557e4de
23 changed files with 488 additions and 269 deletions

View File

@@ -0,0 +1,214 @@
package glossary
import (
"fmt"
"regexp"
"slices"
"sort"
"strings"
"unicode"
"unicode/utf8"
)
type protectedTermDef struct {
identities []Identity
canonical string
}
type protectedOccurrence struct {
text string
canonical string
}
// ProtectedVocabulary is a deterministic glossary-derived protected term set.
type ProtectedVocabulary struct {
termsByFolded map[string]protectedTermDef
pattern *regexp.Regexp
}
// NewProtectedVocabulary prepares the independently selected terms for matching.
func NewProtectedVocabulary(terms []Term) ProtectedVocabulary {
termsByFolded := make(map[string]protectedTermDef, len(terms))
ordered := slices.Clone(terms)
sort.SliceStable(ordered, func(i, j int) bool {
return len(ordered[i].Spelling) > len(ordered[j].Spelling)
})
var groups, alternatives []string
var left, right string
flush := func() {
if len(alternatives) > 0 {
groups = append(groups, left+`(?:`+strings.Join(alternatives, "|")+`)`+right)
alternatives = alternatives[:0]
}
}
for _, term := range ordered {
if term.Spelling == "" {
continue
}
if !term.Exempt {
termsByFolded[strings.ToLower(term.Spelling)] = protectedTermDef{identities: slices.Clone(term.Identities), canonical: term.Spelling}
}
first, _ := utf8.DecodeRuneInString(term.Spelling)
last, _ := utf8.DecodeLastRuneInString(term.Spelling)
// Share boundary checks across adjacent alternatives without
// reordering competing spellings.
a, b := literalEdge(first), literalEdge(last)
if a != left || b != right {
flush()
left, right = a, b
}
alternatives = append(alternatives, regexp.QuoteMeta(term.Spelling))
}
if len(termsByFolded) == 0 {
return ProtectedVocabulary{}
}
flush()
pattern := regexp.MustCompile(`(?i)(?:` + strings.Join(groups, "|") + `)`)
return ProtectedVocabulary{termsByFolded: termsByFolded, pattern: pattern}
}
// A punctuation or symbol edge needs the same ASCII nonword state both
// inside and outside the spelling. Letter and number edges retain Go's
// regexp word-boundary behavior. Underscore is an ASCII word character.
func literalEdge(r rune) string {
if r != '_' && (unicode.IsPunct(r) || unicode.IsSymbol(r)) {
return `\B`
}
return `\b`
}
func (v ProtectedVocabulary) Empty() bool {
return v.pattern == nil
}
// AllowsChange checks the full text of a segment against the protection view.
func (v ProtectedVocabulary) AllowsChange(before, after string, allowRepetitionCleanup bool) bool {
return v.Violation(before, after, allowRepetitionCleanup) == ""
}
// Violation compares usable full-segment text before and after a proposed edit.
// Spoken-word cleanup may reduce repeated mentions while preserving each identity.
func (v ProtectedVocabulary) Violation(before, after string, allowRepetitionCleanup bool) string {
if v.Empty() || before == after {
return ""
}
beforeByID := v.occurrencesByIdentity(before)
afterByID := v.occurrencesByIdentity(after)
if reason := validateIdentityPreservation(beforeByID, afterByID); reason != "" {
if allowRepetitionCleanup && preservesRepeatedIdentities(beforeByID, afterByID) {
return ""
}
return reason
}
return validateCapitalizationTransitions(beforeByID, afterByID)
}
func (v ProtectedVocabulary) occurrencesByIdentity(text string) map[Identity][]protectedOccurrence {
byID := make(map[Identity][]protectedOccurrence)
v.scan(text, func(term protectedTermDef, matched string) {
occurrence := protectedOccurrence{text: matched, canonical: term.canonical}
for _, identity := range term.identities {
byID[identity] = append(byID[identity], occurrence)
}
})
return byID
}
// MentionedIdentities expands each matched spelling's contributors only once.
func (v ProtectedVocabulary) MentionedIdentities(texts ...string) map[Identity]struct{} {
identities := make(map[Identity]struct{})
seen := make(map[string]struct{})
for _, text := range texts {
v.scan(text, func(term protectedTermDef, _ string) {
if _, found := seen[term.canonical]; found {
return
}
seen[term.canonical] = struct{}{}
for _, id := range term.identities {
identities[id] = struct{}{}
}
})
}
return identities
}
func (v ProtectedVocabulary) scan(text string, visit func(protectedTermDef, string)) {
if v.Empty() {
return
}
for _, span := range v.pattern.FindAllStringIndex(text, -1) {
matched := text[span[0]:span[1]]
if term, ok := v.termsByFolded[strings.ToLower(matched)]; ok {
visit(term, matched)
}
}
}
func validateIdentityPreservation(beforeByID, afterByID map[Identity][]protectedOccurrence) string {
var selected string
found := false
for identity, beforeItems := range beforeByID {
afterItems := afterByID[identity]
if len(afterItems) >= len(beforeItems) {
continue
}
// Protect identity counts. Within an identity that lost a mention,
// use spelling counts to identify which alias was affected.
remaining := make(map[string]int)
for _, item := range afterItems {
remaining[item.canonical]++
}
for _, item := range beforeItems {
spelling := item.canonical
if remaining[spelling] > 0 {
remaining[spelling]--
continue
}
if !found || spelling < selected {
selected, found = spelling, true
}
}
}
return protectedSpellingMessage(selected, found)
}
func validateCapitalizationTransitions(beforeByID, afterByID map[Identity][]protectedOccurrence) string {
var selected string
found := false
for identity, afterItems := range afterByID {
beforeItems := beforeByID[identity]
for i, afterItem := range afterItems {
if i >= len(beforeItems) || afterItem.text == beforeItems[i].text || afterItem.text == afterItem.canonical {
continue
}
if !found || afterItem.canonical < selected {
selected, found = afterItem.canonical, true
}
}
}
return protectedSpellingMessage(selected, found)
}
func protectedSpellingMessage(spelling string, found bool) string {
if !found {
return ""
}
return fmt.Sprintf("proposal may alter protected glossary spelling %q", spelling)
}
// preservesRepeatedIdentities permits a spoken-word proposal to reduce repeated
// mentions, but not remove an identity or replace its surviving spelling.
// Editorial and meaning validators still decide whether the cleanup is warranted.
func preservesRepeatedIdentities(original, result map[Identity][]protectedOccurrence) bool {
reduced := false
for identity, occurrences := range original {
remaining := result[identity]
if len(remaining) == 0 {
return false
}
if len(remaining) < len(occurrences) {
reduced = true
}
}
return reduced && validateCapitalizationTransitions(original, result) == ""
}

View File

@@ -0,0 +1,52 @@
package glossary
import "testing"
func TestPunctuationEdgedSpellings(t *testing.T) {
entries := []Entry{{ID: "dotnet", Name: ".NET"}, {ID: "cpp", Name: "C++"}, {ID: "ward", Name: "[Ward]"}}
vocab := NewProtectedVocabulary(Vocabulary(entries, nil))
for _, tc := range []struct {
text, identity string
}{
{".NET", "dotnet"},
{"Using C++.", "cpp"},
{"Take ([Ward]).", "ward"},
{"x.NET", ""},
{"C++like", ""},
{"x[Ward]", ""},
} {
t.Run(tc.text, func(t *testing.T) {
mentioned := vocab.MentionedIdentities(tc.text)
if tc.identity == "" {
if len(mentioned) != 0 || !vocab.AllowsChange(tc.text, "", false) {
t.Fatalf("embedded spelling matched: %v", mentioned)
}
return
}
if _, found := mentioned[Identity{ID: tc.identity}]; !found || len(mentioned) != 1 || vocab.AllowsChange(tc.text, "", false) {
t.Fatalf("punctuation-edged spelling missing: %v", mentioned)
}
})
}
}
func TestSharedSpellingContributesEveryIdentity(t *testing.T) {
entries := []Entry{
{Collection: "a", ID: "first", Name: "First", Aliases: []string{"Shared"}},
{Collection: "b", ID: "second", Name: "Second", Aliases: []string{"Shared"}},
}
vocab := NewProtectedVocabulary(Vocabulary(entries, nil))
ids := vocab.MentionedIdentities("Shared shared", "Shared")
if len(ids) != 2 {
t.Fatalf("mentioned identities: %v", ids)
}
for _, entry := range entries {
id := Identity{Collection: entry.Collection, ID: entry.ID}
if _, found := ids[id]; !found {
t.Errorf("missing identity: %v", id)
}
}
if vocab.AllowsChange("Shared", "First", false) {
t.Fatal("second identity disappeared")
}
}

View File

@@ -151,12 +151,8 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
for i, raw := range response.Corrections {
references = append(references, raw.SegmentReference)
// Unknown references remain invalid candidates for the existing domain validators.
targetID := 0
if req.WorkingTranscript != nil && raw.SegmentReference > 0 && raw.SegmentReference <= len(req.WorkingTranscript.Segments) {
targetID = req.WorkingTranscript.Segments[raw.SegmentReference-1].ID
}
candidate := proposals.CorrectionProposal{
TargetSegmentID: targetID,
TargetSegmentID: proposals.SegmentIDAtReference(req.WorkingTranscript, raw.SegmentReference),
OriginalText: raw.OriginalText,
CorrectedText: raw.CorrectedText,
}

View File

@@ -171,13 +171,14 @@ func TestGenerateCandidatesInvalidCorrectionIsPreservedForLaterValidation(t *tes
responses: []StructuredCorrectionSet{
{
Corrections: []StructuredCorrectionProposal{
{SegmentReference: 0, OriginalText: "x", CorrectedText: ""},
{SegmentReference: 9, OriginalText: "x", CorrectedText: ""},
},
},
},
}
req := defaultRequest(t)
req.LLMClient = client
req.Interactions = llm.NewDiagnosticsWriter(t.TempDir())
result, err := GenerateCandidates(context.Background(), req)
if err != nil {
@@ -189,6 +190,20 @@ func TestGenerateCandidatesInvalidCorrectionIsPreservedForLaterValidation(t *tes
if result.Corrections[0].TargetSegmentID != 0 {
t.Fatalf("unexpected preserved correction: %+v", result.Corrections[0])
}
raw, err := os.ReadFile(result.Artifacts.ResponsePayloadPath)
if err != nil {
t.Fatal(err)
}
var evidence struct {
Decoded StructuredCorrectionSet `json:"decoded"`
}
if err := json.Unmarshal(raw, &evidence); err != nil {
t.Fatal(err)
}
if len(result.SegmentReferences) != 1 || result.SegmentReferences[0] != 9 ||
len(evidence.Decoded.Corrections) != 1 || evidence.Decoded.Corrections[0].SegmentReference != 9 {
t.Fatalf("lost model reference: result=%v artifact=%s", result.SegmentReferences, raw)
}
if len(result.Operations) != 1 || !result.Operations[0].Succeeded {
t.Fatalf("invalid individual correction should remain a usable response: %+v", result.Operations)
}

View File

@@ -170,6 +170,35 @@ func TestApplyProposalsRequireUniqueAmbiguousSkip(t *testing.T) {
}
}
func TestPreviewAndApplyHandleOverlappingOccurrences(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "banana"}}}
edit := EnrichedCorrectionProposal{CorrectionProposal: CorrectionProposal{
TargetSegmentID: 71, OriginalText: "ana", CorrectedText: "X",
}}
for _, tc := range []struct {
policy ReplacementPolicy
text string
skip ProposalSkipReason
}{
{ReplacementPolicyRequireUnique, "banana", SkipReasonAmbiguousOriginal},
{ReplacementPolicyReplaceAll, "bXna", ""},
} {
preview := PreviewProposalForSegment(&transcript.Segments[0], edit.CorrectionProposal, tc.policy)
result := ApplyProposals(transcript, []EnrichedCorrectionProposal{edit}, tc.policy, nil)
if result.Transcript.Segments[0].Text != tc.text {
t.Fatalf("%s produced %q", tc.policy, result.Transcript.Segments[0].Text)
}
if tc.skip != "" {
if preview.HasResult() || preview.SkipReason != tc.skip || len(result.Skipped) != 1 || result.Skipped[0].SkipReason != tc.skip {
t.Fatalf("%s preview %+v and apply %+v disagree with expected skip", tc.policy, preview, result)
}
} else if !preview.Applicable || preview.CorrectedSegmentText != tc.text ||
len(result.Applied) != 1 || result.Applied[0].ReplacementCount != 1 {
t.Fatalf("%s preview %+v and apply %+v disagree with expected edit", tc.policy, preview, result)
}
}
}
func TestApplyProposalsReplaceAllMultipleReplacements(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "uh uh uh"}}}
proposals := []EnrichedCorrectionProposal{

View File

@@ -48,42 +48,34 @@ func PreviewProposalForSegment(segment *schema.Segment, proposal CorrectionPropo
return SegmentPreviewResult{SkipReason: SkipReasonMissingSegment}
}
matchCount := strings.Count(segment.Text, proposal.OriginalText)
if matchCount == 0 {
first := strings.Index(segment.Text, proposal.OriginalText)
if first < 0 {
return SegmentPreviewResult{SkipReason: SkipReasonMissingOriginalText}
}
if proposal.OriginalText == proposal.CorrectedText {
return SegmentPreviewResult{SkipReason: SkipReasonNoEffect}
}
switch policy {
case ReplacementPolicyRequireUnique:
if matchCount != 1 {
count := 1
var corrected string
if policy == ReplacementPolicyRequireUnique {
// Search after the first starting offset, including overlapping matches.
if strings.Contains(segment.Text[first+1:], proposal.OriginalText) {
return SegmentPreviewResult{SkipReason: SkipReasonAmbiguousOriginal}
}
corrected := strings.Replace(segment.Text, proposal.OriginalText, proposal.CorrectedText, 1)
if strings.TrimSpace(corrected) == "" {
return SegmentPreviewResult{SkipReason: SkipReasonEmptyResultingText, CorrectedSegmentText: corrected}
}
return SegmentPreviewResult{
Applicable: true,
CorrectedSegmentText: corrected,
ReplacementCount: 1,
}
case ReplacementPolicyReplaceAll:
corrected := strings.ReplaceAll(segment.Text, proposal.OriginalText, proposal.CorrectedText)
if strings.TrimSpace(corrected) == "" {
return SegmentPreviewResult{SkipReason: SkipReasonEmptyResultingText, CorrectedSegmentText: corrected}
}
return SegmentPreviewResult{
Applicable: true,
CorrectedSegmentText: corrected,
ReplacementCount: matchCount,
}
corrected = strings.Replace(segment.Text, proposal.OriginalText, proposal.CorrectedText, 1)
} else {
count = strings.Count(segment.Text, proposal.OriginalText)
corrected = strings.ReplaceAll(segment.Text, proposal.OriginalText, proposal.CorrectedText)
}
return SegmentPreviewResult{SkipReason: SkipReasonInvalidProposal}
if strings.TrimSpace(corrected) == "" {
return SegmentPreviewResult{SkipReason: SkipReasonEmptyResultingText, CorrectedSegmentText: corrected}
}
return SegmentPreviewResult{Applicable: true, CorrectedSegmentText: corrected, ReplacementCount: count}
}
// HasResult indicates that the replacement was located and yielded a preview,
// including when a separate blank-result rule makes the edit inapplicable.
func (r SegmentPreviewResult) HasResult() bool {
return r.Applicable || r.SkipReason == SkipReasonEmptyResultingText
}

View File

@@ -2,6 +2,8 @@ package proposals
import (
"fmt"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
)
// CorrectionProposal is the base proposal emitted by correction modules.
@@ -39,3 +41,12 @@ func (p CorrectionProposal) Validate() error {
return nil
}
// SegmentIDAtReference maps a one-based model reference in the supplied
// section to the transcript's segment ID. Zero represents an invalid reference.
func SegmentIDAtReference(section *schema.Transcript, reference int) int {
if section == nil || reference <= 0 || reference > len(section.Segments) {
return 0
}
return section.Segments[reference-1].ID
}

View File

@@ -204,12 +204,8 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
}
revision := response.Decision.Correction
localRef = revision.SegmentReference
target := 0
if input.WorkingTranscript != nil && localRef > 0 && localRef <= len(input.WorkingTranscript.Segments) {
target = input.WorkingTranscript.Segments[localRef-1].ID
}
current = original
current.CorrectionProposal = proposals.CorrectionProposal{TargetSegmentID: target, OriginalText: revision.OriginalText, CorrectedText: revision.CorrectedText}
current.CorrectionProposal = proposals.CorrectionProposal{TargetSegmentID: proposals.SegmentIDAtReference(input.WorkingTranscript, localRef), OriginalText: revision.OriginalText, CorrectedText: revision.CorrectedText}
if debug {
branch.Final = current
}

View File

@@ -130,7 +130,9 @@ func TestRepairRequestRefreshesPreviewAfterRetarget(t *testing.T) {
`{"proposal_ref":1,"decision":{"action":"revise","correction":{"id":2,"original_text":"uh ","corrected_text":""}}}`,
`{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`,
}}
var validatedIDs []int
reject := fakeValidator{name: "editorial_review", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
validatedIDs = append(validatedIDs, req.CandidateProposal[0].TargetSegmentID)
return validators.Result{Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Disposition: validators.DispositionRejected, ReasonCode: "llm_rejected", Message: "The context does not support this edit."}}}, nil
}}
input := validateSectionCandidatesInput{
@@ -149,6 +151,9 @@ func TestRepairRequestRefreshesPreviewAfterRetarget(t *testing.T) {
if err != nil || len(client.calls) != 2 || len(branches) != 1 || branches[0].Outcome != "withdrawn" {
t.Fatalf("repair: branches=%+v calls=%d err=%v", branches, len(client.calls), err)
}
if len(validatedIDs) != 2 || validatedIDs[0] != 81 || validatedIDs[1] != 104 {
t.Fatalf("validated segment IDs: %v", validatedIDs)
}
for i, want := range []struct{ before, after string }{{"cat", "dog"}, {"uh hello", "hello"}} {
var payload proposal_generation.RepairRequest
if err := json.Unmarshal([]byte(client.calls[i].Variables["RepairRequestJSON"]), &payload); err != nil {

View File

@@ -9,6 +9,7 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
@@ -290,7 +291,7 @@ func transcriptFromSections(sections []chunking.Section) *schema.Transcript {
// applyWithinSections leaves section membership and initial metadata unchanged.
// Approved proposals are applied only to their identified section.
func applyWithinSections(sections []chunking.Section, approved []proposals.EnrichedCorrectionProposal, policy proposals.ReplacementPolicy, vocabulary validators.ProtectedVocabulary, allowRepetitionCleanup bool) ([]chunking.Section, []proposals.AppliedChange, []proposals.SkippedChange, []int, error) {
func applyWithinSections(sections []chunking.Section, approved []proposals.EnrichedCorrectionProposal, policy proposals.ReplacementPolicy, vocabulary glossary.ProtectedVocabulary, allowRepetitionCleanup bool) ([]chunking.Section, []proposals.AppliedChange, []proposals.SkippedChange, []int, error) {
applied := make([]proposals.AppliedChange, 0)
skipped := make([]proposals.SkippedChange, 0)
appliedBySection := make([]int, len(sections))

View File

@@ -4,6 +4,7 @@ import (
"context"
"strings"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
)
@@ -98,29 +99,25 @@ func (v NoEffectValidator) Validate(_ context.Context, req Request) (Result, err
}
func ValidateProtectedTerms(req Request) (Result, error) {
var vocab ProtectedVocabulary
var vocab glossary.ProtectedVocabulary
if req.GlossaryView != nil {
vocab = req.GlossaryView.Protection
} else {
vocab = NewProtectedVocabulary(req.ProtectedTerms)
vocab = glossary.NewProtectedVocabulary(req.ProtectedTerms)
}
decisions := make([]Decision, 0, len(req.CandidateProposal))
glossaryStage := req.ModuleKey == modulecatalog.KeyGlossary
hasTerms := !vocab.Empty()
if hasTerms {
req.Section = req.sectionLookup()
}
repetitionCleanup := req.ModuleKey == modulecatalog.KeySpokenWord
for _, c := range req.CandidateProposal {
reason := vocab.violationReason(c.OriginalText, c.CorrectedText)
repetitionCleanup := reason != "" && req.ModuleKey == modulecatalog.KeySpokenWord
if vocab.pattern != nil && (glossaryStage || repetitionCleanup) {
req.Section = req.sectionLookup()
var reason string
if hasTerms {
if position, ok := req.Section.Position(c.TargetSegmentID); ok {
segment := &req.Section.segments[position]
preview := req.Preview(c)
if preview.Applicable || (req.IndependentEvidence && preview.SkipReason == proposals.SkipReasonEmptyResultingText) {
if glossaryStage {
// A correction span may sit inside a longer exempt name.
reason = vocab.violationReason(segment.Text, preview.CorrectedSegmentText)
} else if vocab.preservesRepeatedIdentities(segment.Text, preview.CorrectedSegmentText) {
reason = ""
}
if preview.HasResult() {
reason = vocab.Violation(req.Section.segments[position].Text, preview.CorrectedSegmentText, repetitionCleanup)
}
}
}

View File

@@ -11,18 +11,18 @@ import (
// GlossaryView holds the read-only matching data shared by a stage's validators
// and apply-time checks.
type GlossaryView struct {
Protection ProtectedVocabulary
Protection glossary.ProtectedVocabulary
context glossaryContext
}
func PrepareGlossaryView(entries []glossary.Entry, protected []glossary.Term) *GlossaryView {
return &GlossaryView{Protection: NewProtectedVocabulary(protected), context: prepareGlossaryContext(entries)}
return &GlossaryView{Protection: glossary.NewProtectedVocabulary(protected), context: prepareGlossaryContext(entries)}
}
type glossaryContext struct {
entries []glossary.Entry
positions map[glossary.Identity][]int
vocabulary ProtectedVocabulary
vocabulary glossary.ProtectedVocabulary
}
func prepareGlossaryContext(entries []glossary.Entry) glossaryContext {
@@ -31,7 +31,7 @@ func prepareGlossaryContext(entries []glossary.Entry) glossaryContext {
identity := glossary.Identity{Collection: entry.Collection, ID: entry.ID}
positions[identity] = append(positions[identity], i)
}
return glossaryContext{entries: entries, positions: positions, vocabulary: NewProtectedVocabulary(glossary.Vocabulary(entries, nil))}
return glossaryContext{entries: entries, positions: positions, vocabulary: glossary.NewProtectedVocabulary(glossary.Vocabulary(entries, nil))}
}
// validationContext supplies adjacent source segments within the current chunk
@@ -51,12 +51,7 @@ func (view glossaryContext) validationContext(segments []schema.Segment, positio
for _, neighbor := range neighbors {
texts = append(texts, neighbor.Text)
}
mentioned := make(map[glossary.Identity]struct{})
for _, text := range texts {
for _, occurrence := range view.vocabulary.occurrences(text) {
mentioned[occurrence.identity] = struct{}{}
}
}
mentioned := view.vocabulary.MentionedIdentities(texts...)
var matched []int
for identity := range mentioned {
matched = append(matched, view.positions[identity]...)

View File

@@ -1,167 +0,0 @@
package validators
import (
"fmt"
"regexp"
"sort"
"strings"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
)
type protectedTermDef struct {
identities []glossary.Identity
canonical string
}
type protectedOccurrence struct {
text string
identity glossary.Identity
canonical string
}
// ProtectedVocabulary is a deterministic glossary-derived protected term set.
type ProtectedVocabulary struct {
termsByFolded map[string]protectedTermDef
pattern *regexp.Regexp
}
// NewProtectedVocabulary prepares the independently selected terms for matching.
func NewProtectedVocabulary(terms []glossary.Term) ProtectedVocabulary {
termsByFolded := make(map[string]protectedTermDef, len(terms))
alternatives := make([]string, 0, len(terms))
for _, term := range terms {
if !term.Exempt {
termsByFolded[strings.ToLower(term.Spelling)] = protectedTermDef{identities: term.Identities, canonical: term.Spelling}
}
alternatives = append(alternatives, regexp.QuoteMeta(term.Spelling))
}
if len(termsByFolded) == 0 {
return ProtectedVocabulary{}
}
sort.SliceStable(alternatives, func(i, j int) bool { return len(alternatives[i]) > len(alternatives[j]) })
pattern := regexp.MustCompile(`(?i)\b(?:` + strings.Join(alternatives, "|") + `)\b`)
return ProtectedVocabulary{termsByFolded: termsByFolded, pattern: pattern}
}
// AllowsChange checks a proposed segment against the effective protection view.
// Spoken-word cleanup may reduce repeated mentions while preserving each identity.
func (v ProtectedVocabulary) AllowsChange(before, after string, allowRepetitionCleanup bool) bool {
if v.violationReason(before, after) == "" {
return true
}
return allowRepetitionCleanup && v.preservesRepeatedIdentities(before, after)
}
func (v ProtectedVocabulary) violationReason(before, after string) string {
beforeByID := v.occurrencesByIdentity(before)
afterByID := v.occurrencesByIdentity(after)
if reason := validateIdentityPreservation(beforeByID, afterByID); reason != "" {
return reason
}
if reason := validateCapitalizationTransitions(beforeByID, afterByID); reason != "" {
return reason
}
return ""
}
func (v ProtectedVocabulary) occurrencesByIdentity(text string) map[glossary.Identity][]protectedOccurrence {
byID := make(map[glossary.Identity][]protectedOccurrence)
for _, o := range v.occurrences(text) {
byID[o.identity] = append(byID[o.identity], o)
}
return byID
}
func (v ProtectedVocabulary) occurrences(text string) []protectedOccurrence {
if v.pattern == nil {
return nil
}
matches := v.pattern.FindAllStringIndex(text, -1)
if len(matches) == 0 {
return nil
}
out := make([]protectedOccurrence, 0, len(matches))
for _, idx := range matches {
matched := text[idx[0]:idx[1]]
def, ok := v.termsByFolded[strings.ToLower(matched)]
if !ok {
continue
}
for _, identity := range def.identities {
out = append(out, protectedOccurrence{text: matched, identity: identity, canonical: def.canonical})
}
}
return out
}
func validateIdentityPreservation(beforeByID, afterByID map[glossary.Identity][]protectedOccurrence) string {
var selected string
found := false
for identity, beforeItems := range beforeByID {
afterItems := afterByID[identity]
if len(afterItems) >= len(beforeItems) {
continue
}
// Protect identity counts. Within an identity that lost a mention,
// use spelling counts to identify which alias was affected.
remaining := make(map[string]int)
for _, item := range afterItems {
remaining[item.canonical]++
}
for _, item := range beforeItems {
spelling := item.canonical
if remaining[spelling] > 0 {
remaining[spelling]--
continue
}
if !found || spelling < selected {
selected, found = spelling, true
}
}
}
return protectedSpellingMessage(selected, found)
}
func validateCapitalizationTransitions(beforeByID, afterByID map[glossary.Identity][]protectedOccurrence) string {
var selected string
found := false
for identity, afterItems := range afterByID {
beforeItems := beforeByID[identity]
for i, afterItem := range afterItems {
if i >= len(beforeItems) || afterItem.text == beforeItems[i].text || afterItem.text == afterItem.canonical {
continue
}
if !found || afterItem.canonical < selected {
selected, found = afterItem.canonical, true
}
}
}
return protectedSpellingMessage(selected, found)
}
func protectedSpellingMessage(spelling string, found bool) string {
if !found {
return ""
}
return fmt.Sprintf("proposal may alter protected glossary spelling %q", spelling)
}
// preservesRepeatedIdentities permits a spoken-word proposal to reduce repeated
// mentions, but not remove an identity or replace its surviving spelling.
// Editorial and meaning validators still decide whether the cleanup is warranted.
func (v ProtectedVocabulary) preservesRepeatedIdentities(before, after string) bool {
original := v.occurrencesByIdentity(before)
result := v.occurrencesByIdentity(after)
reduced := false
for identity, occurrences := range original {
remaining := result[identity]
if len(remaining) == 0 {
return false
}
if len(remaining) < len(occurrences) {
reduced = true
}
}
return reduced && validateCapitalizationTransitions(original, result) == ""
}

View File

@@ -30,9 +30,11 @@ func TestRecognizedTermsDeduplicateAndIncludePlurals(t *testing.T) {
func TestValidateProtectedTermsAppliesToNonGlossaryModule(t *testing.T) {
req := Request{
ModuleKey: "grammar",
Glossary: []domain.Entry{{ID: "jesters", Name: "Jesters", Category: "faction"}},
ProtectedTerms: domain.Vocabulary([]domain.Entry{{ID: "jesters", Name: "Jesters"}}, nil),
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Jesters"}}},
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
ModuleKey: "grammar",
Glossary: []domain.Entry{{ID: "jesters", Name: "Jesters", Category: "faction"}},
ProtectedTerms: domain.Vocabulary([]domain.Entry{{ID: "jesters", Name: "Jesters"}}, nil),
CandidateProposal: []proposals.EnrichedCorrectionProposal{
{
CorrectionProposal: proposals.CorrectionProposal{
@@ -130,3 +132,32 @@ func TestProtectedNameFindingIdentifiesRemovedAlias(t *testing.T) {
t.Fatalf("finding did not identify the removed alias: %+v", decision)
}
}
func TestProtectionRequiresAUsableFullSegmentPreview(t *testing.T) {
for _, tc := range []struct {
name, source, span, replacement string
target int
rejected bool
}{
{"partial name change", "Cast Mage Armor.", "Mage", "Magus", 1, true},
{"ambiguous span", "Mage Armor and Mage Armor", "Mage Armor", "ward", 1, false},
{"missing span", "Jesters arrived", "Mage Armor", "ward", 1, false},
{"missing target", "Mage Armor", "Mage Armor", "ward", 9, false},
{"empty result", "Mage Armor", "Mage Armor", "", 1, true},
} {
t.Run(tc.name, func(t *testing.T) {
req := Request{
ModuleKey: "grammar", ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
ProtectedTerms: domain.Vocabulary([]domain.Entry{{ID: "mage", Name: "Mage Armor"}}, nil),
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: tc.source}}},
CandidateProposal: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{
TargetSegmentID: tc.target, OriginalText: tc.span, CorrectedText: tc.replacement,
}}},
}
result, err := ValidateProtectedTerms(req)
if err != nil || len(result.Decisions) != 1 || result.Decisions[0].Approved == tc.rejected {
t.Fatalf("protection decision = %+v (%v)", result, err)
}
})
}
}

View File

@@ -140,9 +140,11 @@ func TestNoEffectValidator(t *testing.T) {
func TestValidateProtectedTerms(t *testing.T) {
entries := []domain.Entry{{ID: "openai", Name: "OpenAI", Aliases: []string{"Open AI"}, Category: "brand", Description: "brand"}}
req := Request{
Glossary: entries,
ProtectedTerms: domain.Vocabulary(entries, nil),
ModuleKey: "grammar",
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "OpenAI teh"}}},
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
Glossary: entries,
ProtectedTerms: domain.Vocabulary(entries, nil),
ModuleKey: "grammar",
CandidateProposal: []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "OpenAI", "Open A I"),
mkCandidate(1, 1, "teh", "the"),

View File

@@ -24,7 +24,11 @@ func TestIndependentProtectionForSelectedGlossary(t *testing.T) {
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
result, err := ValidateProtectedTerms(Request{ProtectedTerms: tc.terms, CandidateProposal: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{OriginalText: tc.before, CorrectedText: tc.after}}}})
result, err := ValidateProtectedTerms(Request{
ProtectedTerms: tc.terms, ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: tc.before}}},
CandidateProposal: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: tc.before, CorrectedText: tc.after}}},
})
if err != nil || len(result.Decisions) != 1 || result.Decisions[0].Approved != tc.approved {
t.Fatalf("result=%+v err=%v", result, err)
}
@@ -32,22 +36,10 @@ func TestIndependentProtectionForSelectedGlossary(t *testing.T) {
}
}
func TestSharedAliasRetainsAllIdentities(t *testing.T) {
entries := []glossary.Entry{
{Collection: "a", ID: "first", Name: "First", Aliases: []string{"Shared"}},
{Collection: "b", ID: "second", Name: "Second", Aliases: []string{"Shared"}},
}
vocabulary := NewProtectedVocabulary(glossary.Vocabulary(entries, nil))
identities := vocabulary.occurrencesByIdentity("Shared")
if len(identities) != 2 || len(identities[glossary.Identity{Collection: "a", ID: "first"}]) != 1 || len(identities[glossary.Identity{Collection: "b", ID: "second"}]) != 1 {
t.Fatalf("identities=%v", identities)
}
}
func TestApplicationChecksCurrentTextWithSelectedProtection(t *testing.T) {
active := []glossary.Entry{{Collection: "spells", ID: "spell", Name: "Shield"}}
protected := append(append([]glossary.Entry(nil), active...), glossary.Entry{Collection: "items", ID: "item", Name: "Shield", Aliases: []string{"Aegis"}})
vocabulary := NewProtectedVocabulary(glossary.Vocabulary(protected, active))
vocabulary := glossary.NewProtectedVocabulary(glossary.Vocabulary(protected, active))
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Shield and Aegis"}}}
candidates := []proposals.EnrichedCorrectionProposal{
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "Shield", CorrectedText: "Barrier"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}},
@@ -61,6 +53,32 @@ func TestApplicationChecksCurrentTextWithSelectedProtection(t *testing.T) {
}
}
func TestApplicationRechecksProtectionAfterEarlierEdit(t *testing.T) {
active := []glossary.Entry{{ID: "greater", Name: "Greater Mage Armor"}}
terms := glossary.Vocabulary(append(active, glossary.Entry{ID: "short", Name: "Mage Armor"}), active)
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Cast Greater Mage Armor."}}}
candidates := []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "Greater ", ""),
mkCandidate(1, 1, "Mage Armor", "Magus Armor"),
}
policy := proposals.ReplacementPolicyRequireUnique
validation, err := ValidateProtectedTerms(Request{
ModuleKey: "glossary", WorkingTranscript: transcript, ReplacementPolicy: policy,
ProtectedTerms: terms, CandidateProposal: candidates,
})
if err != nil || len(validation.Decisions) != 2 || !validation.Decisions[0].Approved || !validation.Decisions[1].Approved {
t.Fatalf("initial decisions = %+v (%v)", validation, err)
}
vocabulary := glossary.NewProtectedVocabulary(terms)
result := proposals.ApplyProposals(transcript, candidates, policy, func(before, after string) bool {
return vocabulary.AllowsChange(before, after, false)
})
if result.Transcript.Segments[0].Text != "Cast Mage Armor." || len(result.Applied) != 1 ||
len(result.Skipped) != 1 || result.Skipped[0].ProposalIndex != 1 || result.Skipped[0].SkipReason != proposals.SkipReasonProtectedTerm {
t.Fatalf("stale proposal outcome: %+v", result)
}
}
func TestExemptGlossarySpellingKeepsLongestMatchBoundary(t *testing.T) {
active := []glossary.Entry{
{ID: "greater-mage", Name: "Greater Mage Armor"},
@@ -96,7 +114,7 @@ func TestExemptGlossarySpellingKeepsLongestMatchBoundary(t *testing.T) {
if err != nil || len(decision.Decisions) != 1 || decision.Decisions[0].Approved != tc.allowed {
t.Fatalf("decision=%+v err=%v", decision, err)
}
vocabulary := NewProtectedVocabulary(terms)
vocabulary := glossary.NewProtectedVocabulary(terms)
result := proposals.ApplyProposals(transcript, []proposals.EnrichedCorrectionProposal{candidate}, policy, func(before, after string) bool {
return vocabulary.AllowsChange(before, after, false)
})

View File

@@ -170,6 +170,7 @@ func TestHomophonesProtectedGlossaryTermBehaviorRejectsUnsafeCorrection(t *testi
}}
entries := tinyGlossary()
result, err := protectedValidator.Validate(context.Background(), validators.Request{
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
WorkingTranscript: working,
Glossary: entries,
ProtectedTerms: domain.Vocabulary(entries, nil),

View File

@@ -170,6 +170,7 @@ func TestSpokenWordProtectedGlossaryTermBehaviorRejectsUnsafeCorrection(t *testi
}}
entries := tinyGlossary()
result, err := protectedValidator.Validate(context.Background(), validators.Request{
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
WorkingTranscript: working,
Glossary: entries,
ProtectedTerms: domain.Vocabulary(entries, nil),

View File

@@ -5,6 +5,7 @@ import (
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
@@ -78,9 +79,11 @@ func TestNewRejectsProtectedTermChangeOutsideGlossaryStage(t *testing.T) {
}
req := contracts.ValidationRequest{
ModuleKey: "homophones",
Glossary: tinyGlossary(),
ProtectedTerms: glossary.Vocabulary(tinyGlossary(), nil),
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Jesters"}}},
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
ModuleKey: "homophones",
Glossary: tinyGlossary(),
ProtectedTerms: glossary.Vocabulary(tinyGlossary(), nil),
CandidateProposal: []proposals.EnrichedCorrectionProposal{
proposal(0, "homophones", "Jesters", "Gestures"),
},
@@ -113,9 +116,11 @@ func TestNewGlossaryStageRejectsProtectedTermChange(t *testing.T) {
}
req := contracts.ValidationRequest{
ModuleKey: "glossary",
Glossary: tinyGlossary(),
ProtectedTerms: glossary.Vocabulary(tinyGlossary(), nil),
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Jesters"}}},
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
ModuleKey: "glossary",
Glossary: tinyGlossary(),
ProtectedTerms: glossary.Vocabulary(tinyGlossary(), nil),
CandidateProposal: []proposals.EnrichedCorrectionProposal{
proposal(0, "glossary", "Jesters", "Gestures"),
},
@@ -142,9 +147,11 @@ func TestNewGlossaryStageAllowsProtectedTermInsertion(t *testing.T) {
}
req := contracts.ValidationRequest{
ModuleKey: "glossary",
Glossary: tinyGlossary(),
ProtectedTerms: glossary.Vocabulary(tinyGlossary(), nil),
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "gestures"}}},
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
ModuleKey: "glossary",
Glossary: tinyGlossary(),
ProtectedTerms: glossary.Vocabulary(tinyGlossary(), nil),
CandidateProposal: []proposals.EnrichedCorrectionProposal{
proposal(0, "glossary", "gestures", "Jesters"),
},

View File

@@ -6,6 +6,7 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/validators/editorial_review"
@@ -163,9 +164,11 @@ func TestResolveBuiltInChainUsesGlossaryStageProtectedTermsBehavior(t *testing.T
}
req := contracts.ValidationRequest{
ModuleKey: "glossary",
ProtectedTerms: glossary.Vocabulary([]glossary.Entry{{Collection: "party", ID: "jesters", Name: "Jesters"}}, nil),
Glossary: []glossary.Entry{{Collection: "party", ID: "jesters", Name: "Jesters", Category: "faction", Description: "Protected"}},
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Jesters"}}},
ReplacementPolicy: proposals.ReplacementPolicyReplaceAll,
ModuleKey: "glossary",
ProtectedTerms: glossary.Vocabulary([]glossary.Entry{{Collection: "party", ID: "jesters", Name: "Jesters"}}, nil),
Glossary: []glossary.Entry{{Collection: "party", ID: "jesters", Name: "Jesters", Category: "faction", Description: "Protected"}},
CandidateProposal: []proposals.EnrichedCorrectionProposal{
{
CorrectionProposal: proposals.CorrectionProposal{