Share edit location and check protected names in full segments

This commit is contained in:
2026-11-16 04:09:25 +00:00
parent 43ce620122
commit 475557e4de
23 changed files with 488 additions and 269 deletions

View File

@@ -46,7 +46,7 @@ for each request. PromptKit renders the selected definition and validates the
correction-set schema before Audita maps proposals. Domain validators still
check applicability, protected glossary terms, and replacement
policy. Model-facing segment IDs are one-based positions within the proposal
request. Generation maps returned references back to the current chunk's actual
request. Generation and repair map those references back to the current chunk's actual
segment IDs, preserving response order and multiple proposals per segment.
Unknown references become invalid-target candidates for the existing domain
checks; they are never interpreted as actual transcript IDs. Debug request

View File

@@ -155,7 +155,19 @@ counts for fitting and batching. Context is supplemental, not a promise that eve
relevant definition or neighboring utterance always fits. This uses the existing
payload estimator, not a new guarantee about the provider's full context window.
## Protected-Name Repetition
## Protected Names
Protection compares the source segment with the full replacement preview.
Missing and ambiguous spans have no protection verdict. An empty preview can
produce a protection finding alongside the blank-result rule. Application
rechecks the preview against the current segment after earlier edits.
The glossary matcher selects the longest spelling at a match position.
Punctuation-edged spellings use literal edge checks with Go's ASCII regexp
boundary semantics. Active glossary exemptions take part in match selection.
Validation context uses the same scan to select identities in glossary order.
### Protected-Name Repetition
Spoken-word cleanup may reduce repeated protected-name occurrences when every
original protected identity still appears in the resulting segment and surviving
@@ -164,4 +176,4 @@ both a replacement of the repeated phrase and deletion of a redundant span. This
exception only relaxes the deterministic count guard; editorial and meaning-reversal
validation still decide whether the repetition was accidental and the edit preserves
meaning. Intentional emphasis and substantive references remain protected by those
reviews. Other modules retain their existing protection rules.
reviews.

View File

@@ -312,6 +312,14 @@ ambiguous previews, spoken-word cleanup, and stale apply-time changes at the
owning boundaries. Check the initial and repair lookup against independent
expected references, and compare work on a repeated-glossary fixture.
**Complete.** Proposal preview and application agree on overlapping spans.
Initial and repair responses share positional lookup and retain original model
references in diagnostic evidence. Protection runs on usable full-segment
previews and checks the current text at application. The 2,000-mention
fixture with 128 shared contributors measured 81.0 MB to 0.27 MB allocated
and 77.8 ms to 2.1 ms per operation (Go 1.27.3, ten iterations). The CLI build
and full offline suite passed.
## Stage 9 — Consolidate validator decisions and reconciliation
Audit #10, #11, #27 (runtime class), #40, #49 (validator port).

View File

@@ -0,0 +1,214 @@
package glossary
import (
"fmt"
"regexp"
"slices"
"sort"
"strings"
"unicode"
"unicode/utf8"
)
type protectedTermDef struct {
identities []Identity
canonical string
}
type protectedOccurrence struct {
text string
canonical string
}
// ProtectedVocabulary is a deterministic glossary-derived protected term set.
type ProtectedVocabulary struct {
termsByFolded map[string]protectedTermDef
pattern *regexp.Regexp
}
// NewProtectedVocabulary prepares the independently selected terms for matching.
func NewProtectedVocabulary(terms []Term) ProtectedVocabulary {
termsByFolded := make(map[string]protectedTermDef, len(terms))
ordered := slices.Clone(terms)
sort.SliceStable(ordered, func(i, j int) bool {
return len(ordered[i].Spelling) > len(ordered[j].Spelling)
})
var groups, alternatives []string
var left, right string
flush := func() {
if len(alternatives) > 0 {
groups = append(groups, left+`(?:`+strings.Join(alternatives, "|")+`)`+right)
alternatives = alternatives[:0]
}
}
for _, term := range ordered {
if term.Spelling == "" {
continue
}
if !term.Exempt {
termsByFolded[strings.ToLower(term.Spelling)] = protectedTermDef{identities: slices.Clone(term.Identities), canonical: term.Spelling}
}
first, _ := utf8.DecodeRuneInString(term.Spelling)
last, _ := utf8.DecodeLastRuneInString(term.Spelling)
// Share boundary checks across adjacent alternatives without
// reordering competing spellings.
a, b := literalEdge(first), literalEdge(last)
if a != left || b != right {
flush()
left, right = a, b
}
alternatives = append(alternatives, regexp.QuoteMeta(term.Spelling))
}
if len(termsByFolded) == 0 {
return ProtectedVocabulary{}
}
flush()
pattern := regexp.MustCompile(`(?i)(?:` + strings.Join(groups, "|") + `)`)
return ProtectedVocabulary{termsByFolded: termsByFolded, pattern: pattern}
}
// A punctuation or symbol edge needs the same ASCII nonword state both
// inside and outside the spelling. Letter and number edges retain Go's
// regexp word-boundary behavior. Underscore is an ASCII word character.
func literalEdge(r rune) string {
if r != '_' && (unicode.IsPunct(r) || unicode.IsSymbol(r)) {
return `\B`
}
return `\b`
}
func (v ProtectedVocabulary) Empty() bool {
return v.pattern == nil
}
// AllowsChange checks the full text of a segment against the protection view.
func (v ProtectedVocabulary) AllowsChange(before, after string, allowRepetitionCleanup bool) bool {
return v.Violation(before, after, allowRepetitionCleanup) == ""
}
// Violation compares usable full-segment text before and after a proposed edit.
// Spoken-word cleanup may reduce repeated mentions while preserving each identity.
func (v ProtectedVocabulary) Violation(before, after string, allowRepetitionCleanup bool) string {
if v.Empty() || before == after {
return ""
}
beforeByID := v.occurrencesByIdentity(before)
afterByID := v.occurrencesByIdentity(after)
if reason := validateIdentityPreservation(beforeByID, afterByID); reason != "" {
if allowRepetitionCleanup && preservesRepeatedIdentities(beforeByID, afterByID) {
return ""
}
return reason
}
return validateCapitalizationTransitions(beforeByID, afterByID)
}
func (v ProtectedVocabulary) occurrencesByIdentity(text string) map[Identity][]protectedOccurrence {
byID := make(map[Identity][]protectedOccurrence)
v.scan(text, func(term protectedTermDef, matched string) {
occurrence := protectedOccurrence{text: matched, canonical: term.canonical}
for _, identity := range term.identities {
byID[identity] = append(byID[identity], occurrence)
}
})
return byID
}
// MentionedIdentities expands each matched spelling's contributors only once.
func (v ProtectedVocabulary) MentionedIdentities(texts ...string) map[Identity]struct{} {
identities := make(map[Identity]struct{})
seen := make(map[string]struct{})
for _, text := range texts {
v.scan(text, func(term protectedTermDef, _ string) {
if _, found := seen[term.canonical]; found {
return
}
seen[term.canonical] = struct{}{}
for _, id := range term.identities {
identities[id] = struct{}{}
}
})
}
return identities
}
func (v ProtectedVocabulary) scan(text string, visit func(protectedTermDef, string)) {
if v.Empty() {
return
}
for _, span := range v.pattern.FindAllStringIndex(text, -1) {
matched := text[span[0]:span[1]]
if term, ok := v.termsByFolded[strings.ToLower(matched)]; ok {
visit(term, matched)
}
}
}
func validateIdentityPreservation(beforeByID, afterByID map[Identity][]protectedOccurrence) string {
var selected string
found := false
for identity, beforeItems := range beforeByID {
afterItems := afterByID[identity]
if len(afterItems) >= len(beforeItems) {
continue
}
// Protect identity counts. Within an identity that lost a mention,
// use spelling counts to identify which alias was affected.
remaining := make(map[string]int)
for _, item := range afterItems {
remaining[item.canonical]++
}
for _, item := range beforeItems {
spelling := item.canonical
if remaining[spelling] > 0 {
remaining[spelling]--
continue
}
if !found || spelling < selected {
selected, found = spelling, true
}
}
}
return protectedSpellingMessage(selected, found)
}
func validateCapitalizationTransitions(beforeByID, afterByID map[Identity][]protectedOccurrence) string {
var selected string
found := false
for identity, afterItems := range afterByID {
beforeItems := beforeByID[identity]
for i, afterItem := range afterItems {
if i >= len(beforeItems) || afterItem.text == beforeItems[i].text || afterItem.text == afterItem.canonical {
continue
}
if !found || afterItem.canonical < selected {
selected, found = afterItem.canonical, true
}
}
}
return protectedSpellingMessage(selected, found)
}
func protectedSpellingMessage(spelling string, found bool) string {
if !found {
return ""
}
return fmt.Sprintf("proposal may alter protected glossary spelling %q", spelling)
}
// preservesRepeatedIdentities permits a spoken-word proposal to reduce repeated
// mentions, but not remove an identity or replace its surviving spelling.
// Editorial and meaning validators still decide whether the cleanup is warranted.
func preservesRepeatedIdentities(original, result map[Identity][]protectedOccurrence) bool {
reduced := false
for identity, occurrences := range original {
remaining := result[identity]
if len(remaining) == 0 {
return false
}
if len(remaining) < len(occurrences) {
reduced = true
}
}
return reduced && validateCapitalizationTransitions(original, result) == ""
}

View File

@@ -0,0 +1,52 @@
package glossary
import "testing"
func TestPunctuationEdgedSpellings(t *testing.T) {
entries := []Entry{{ID: "dotnet", Name: ".NET"}, {ID: "cpp", Name: "C++"}, {ID: "ward", Name: "[Ward]"}}
vocab := NewProtectedVocabulary(Vocabulary(entries, nil))
for _, tc := range []struct {
text, identity string
}{
{".NET", "dotnet"},
{"Using C++.", "cpp"},
{"Take ([Ward]).", "ward"},
{"x.NET", ""},
{"C++like", ""},
{"x[Ward]", ""},
} {
t.Run(tc.text, func(t *testing.T) {
mentioned := vocab.MentionedIdentities(tc.text)
if tc.identity == "" {
if len(mentioned) != 0 || !vocab.AllowsChange(tc.text, "", false) {
t.Fatalf("embedded spelling matched: %v", mentioned)
}
return
}
if _, found := mentioned[Identity{ID: tc.identity}]; !found || len(mentioned) != 1 || vocab.AllowsChange(tc.text, "", false) {
t.Fatalf("punctuation-edged spelling missing: %v", mentioned)
}
})
}
}
func TestSharedSpellingContributesEveryIdentity(t *testing.T) {
entries := []Entry{
{Collection: "a", ID: "first", Name: "First", Aliases: []string{"Shared"}},
{Collection: "b", ID: "second", Name: "Second", Aliases: []string{"Shared"}},
}
vocab := NewProtectedVocabulary(Vocabulary(entries, nil))
ids := vocab.MentionedIdentities("Shared shared", "Shared")
if len(ids) != 2 {
t.Fatalf("mentioned identities: %v", ids)
}
for _, entry := range entries {
id := Identity{Collection: entry.Collection, ID: entry.ID}
if _, found := ids[id]; !found {
t.Errorf("missing identity: %v", id)
}
}
if vocab.AllowsChange("Shared", "First", false) {
t.Fatal("second identity disappeared")
}
}

View File

@@ -151,12 +151,8 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
for i, raw := range response.Corrections {
references = append(references, raw.SegmentReference)
// Unknown references remain invalid candidates for the existing domain validators.
targetID := 0
if req.WorkingTranscript != nil && raw.SegmentReference > 0 && raw.SegmentReference <= len(req.WorkingTranscript.Segments) {
targetID = req.WorkingTranscript.Segments[raw.SegmentReference-1].ID
}
candidate := proposals.CorrectionProposal{
TargetSegmentID: targetID,
TargetSegmentID: proposals.SegmentIDAtReference(req.WorkingTranscript, raw.SegmentReference),
OriginalText: raw.OriginalText,
CorrectedText: raw.CorrectedText,
}

View File

@@ -171,13 +171,14 @@ func TestGenerateCandidatesInvalidCorrectionIsPreservedForLaterValidation(t *tes
responses: []StructuredCorrectionSet{
{
Corrections: []StructuredCorrectionProposal{
{SegmentReference: 0, OriginalText: "x", CorrectedText: ""},
{SegmentReference: 9, OriginalText: "x", CorrectedText: ""},
},
},
},
}
req := defaultRequest(t)
req.LLMClient = client
req.Interactions = llm.NewDiagnosticsWriter(t.TempDir())
result, err := GenerateCandidates(context.Background(), req)
if err != nil {
@@ -189,6 +190,20 @@ func TestGenerateCandidatesInvalidCorrectionIsPreservedForLaterValidation(t *tes
if result.Corrections[0].TargetSegmentID != 0 {
t.Fatalf("unexpected preserved correction: %+v", result.Corrections[0])
}
raw, err := os.ReadFile(result.Artifacts.ResponsePayloadPath)
if err != nil {
t.Fatal(err)
}
var evidence struct {
Decoded StructuredCorrectionSet `json:"decoded"`
}
if err := json.Unmarshal(raw, &evidence); err != nil {
t.Fatal(err)
}
if len(result.SegmentReferences) != 1 || result.SegmentReferences[0] != 9 ||
len(evidence.Decoded.Corrections) != 1 || evidence.Decoded.Corrections[0].SegmentReference != 9 {
t.Fatalf("lost model reference: result=%v artifact=%s", result.SegmentReferences, raw)
}
if len(result.Operations) != 1 || !result.Operations[0].Succeeded {
t.Fatalf("invalid individual correction should remain a usable response: %+v", result.Operations)
}

View File

@@ -170,6 +170,35 @@ func TestApplyProposalsRequireUniqueAmbiguousSkip(t *testing.T) {
}
}
func TestPreviewAndApplyHandleOverlappingOccurrences(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "banana"}}}
edit := EnrichedCorrectionProposal{CorrectionProposal: CorrectionProposal{
TargetSegmentID: 71, OriginalText: "ana", CorrectedText: "X",
}}
for _, tc := range []struct {
policy ReplacementPolicy
text string
skip ProposalSkipReason
}{
{ReplacementPolicyRequireUnique, "banana", SkipReasonAmbiguousOriginal},
{ReplacementPolicyReplaceAll, "bXna", ""},
} {
preview := PreviewProposalForSegment(&transcript.Segments[0], edit.CorrectionProposal, tc.policy)
result := ApplyProposals(transcript, []EnrichedCorrectionProposal{edit}, tc.policy, nil)
if result.Transcript.Segments[0].Text != tc.text {
t.Fatalf("%s produced %q", tc.policy, result.Transcript.Segments[0].Text)
}
if tc.skip != "" {
if preview.HasResult() || preview.SkipReason != tc.skip || len(result.Skipped) != 1 || result.Skipped[0].SkipReason != tc.skip {
t.Fatalf("%s preview %+v and apply %+v disagree with expected skip", tc.policy, preview, result)
}
} else if !preview.Applicable || preview.CorrectedSegmentText != tc.text ||
len(result.Applied) != 1 || result.Applied[0].ReplacementCount != 1 {
t.Fatalf("%s preview %+v and apply %+v disagree with expected edit", tc.policy, preview, result)
}
}
}
func TestApplyProposalsReplaceAllMultipleReplacements(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "uh uh uh"}}}
proposals := []EnrichedCorrectionProposal{

View File

@@ -48,42 +48,34 @@ func PreviewProposalForSegment(segment *schema.Segment, proposal CorrectionPropo
return SegmentPreviewResult{SkipReason: SkipReasonMissingSegment}
}
matchCount := strings.Count(segment.Text, proposal.OriginalText)
if matchCount == 0 {
first := strings.Index(segment.Text, proposal.OriginalText)
if first < 0 {
return SegmentPreviewResult{SkipReason: SkipReasonMissingOriginalText}
}
if proposal.OriginalText == proposal.CorrectedText {
return SegmentPreviewResult{SkipReason: SkipReasonNoEffect}
}
switch policy {
case ReplacementPolicyRequireUnique:
if matchCount != 1 {
count := 1
var corrected string
if policy == ReplacementPolicyRequireUnique {
// Search after the first starting offset, including overlapping matches.
if strings.Contains(segment.Text[first+1:], proposal.OriginalText) {
return SegmentPreviewResult{SkipReason: SkipReasonAmbiguousOriginal}
}
corrected := strings.Replace(segment.Text, proposal.OriginalText, proposal.CorrectedText, 1)
if strings.TrimSpace(corrected) == "" {
return SegmentPreviewResult{SkipReason: SkipReasonEmptyResultingText, CorrectedSegmentText: corrected}
}
return SegmentPreviewResult{
Applicable: true,
CorrectedSegmentText: corrected,
ReplacementCount: 1,
}
case ReplacementPolicyReplaceAll:
corrected := strings.ReplaceAll(segment.Text, proposal.OriginalText, proposal.CorrectedText)
if strings.TrimSpace(corrected) == "" {
return SegmentPreviewResult{SkipReason: SkipReasonEmptyResultingText, CorrectedSegmentText: corrected}
}
return SegmentPreviewResult{
Applicable: true,
CorrectedSegmentText: corrected,
ReplacementCount: matchCount,
}
corrected = strings.Replace(segment.Text, proposal.OriginalText, proposal.CorrectedText, 1)
} else {
count = strings.Count(segment.Text, proposal.OriginalText)
corrected = strings.ReplaceAll(segment.Text, proposal.OriginalText, proposal.CorrectedText)
}
return SegmentPreviewResult{SkipReason: SkipReasonInvalidProposal}
if strings.TrimSpace(corrected) == "" {
return SegmentPreviewResult{SkipReason: SkipReasonEmptyResultingText, CorrectedSegmentText: corrected}
}
return SegmentPreviewResult{Applicable: true, CorrectedSegmentText: corrected, ReplacementCount: count}
}
// HasResult indicates that the replacement was located and yielded a preview,
// including when a separate blank-result rule makes the edit inapplicable.
func (r SegmentPreviewResult) HasResult() bool {
return r.Applicable || r.SkipReason == SkipReasonEmptyResultingText
}

View File

@@ -2,6 +2,8 @@ package proposals
import (
"fmt"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
)
// CorrectionProposal is the base proposal emitted by correction modules.
@@ -39,3 +41,12 @@ func (p CorrectionProposal) Validate() error {
return nil
}
// SegmentIDAtReference maps a one-based model reference in the supplied
// section to the transcript's segment ID. Zero represents an invalid reference.
func SegmentIDAtReference(section *schema.Transcript, reference int) int {
if section == nil || reference <= 0 || reference > len(section.Segments) {
return 0
}
return section.Segments[reference-1].ID
}

View File

@@ -204,12 +204,8 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
}
revision := response.Decision.Correction
localRef = revision.SegmentReference
target := 0
if input.WorkingTranscript != nil && localRef > 0 && localRef <= len(input.WorkingTranscript.Segments) {
target = input.WorkingTranscript.Segments[localRef-1].ID
}
current = original
current.CorrectionProposal = proposals.CorrectionProposal{TargetSegmentID: target, OriginalText: revision.OriginalText, CorrectedText: revision.CorrectedText}
current.CorrectionProposal = proposals.CorrectionProposal{TargetSegmentID: proposals.SegmentIDAtReference(input.WorkingTranscript, localRef), OriginalText: revision.OriginalText, CorrectedText: revision.CorrectedText}
if debug {
branch.Final = current
}

View File

@@ -130,7 +130,9 @@ func TestRepairRequestRefreshesPreviewAfterRetarget(t *testing.T) {
`{"proposal_ref":1,"decision":{"action":"revise","correction":{"id":2,"original_text":"uh ","corrected_text":""}}}`,
`{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`,
}}
var validatedIDs []int
reject := fakeValidator{name: "editorial_review", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
validatedIDs = append(validatedIDs, req.CandidateProposal[0].TargetSegmentID)
return validators.Result{Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Disposition: validators.DispositionRejected, ReasonCode: "llm_rejected", Message: "The context does not support this edit."}}}, nil
}}
input := validateSectionCandidatesInput{
@@ -149,6 +151,9 @@ func TestRepairRequestRefreshesPreviewAfterRetarget(t *testing.T) {
if err != nil || len(client.calls) != 2 || len(branches) != 1 || branches[0].Outcome != "withdrawn" {
t.Fatalf("repair: branches=%+v calls=%d err=%v", branches, len(client.calls), err)
}
if len(validatedIDs) != 2 || validatedIDs[0] != 81 || validatedIDs[1] != 104 {
t.Fatalf("validated segment IDs: %v", validatedIDs)
}
for i, want := range []struct{ before, after string }{{"cat", "dog"}, {"uh hello", "hello"}} {
var payload proposal_generation.RepairRequest
if err := json.Unmarshal([]byte(client.calls[i].Variables["RepairRequestJSON"]), &payload); err != nil {

View File

@@ -9,6 +9,7 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
@@ -290,7 +291,7 @@ func transcriptFromSections(sections []chunking.Section) *schema.Transcript {
// applyWithinSections leaves section membership and initial metadata unchanged.
// Approved proposals are applied only to their identified section.
func applyWithinSections(sections []chunking.Section, approved []proposals.EnrichedCorrectionProposal, policy proposals.ReplacementPolicy, vocabulary validators.ProtectedVocabulary, allowRepetitionCleanup bool) ([]chunking.Section, []proposals.AppliedChange, []proposals.SkippedChange, []int, error) {
func applyWithinSections(sections []chunking.Section, approved []proposals.EnrichedCorrectionProposal, policy proposals.ReplacementPolicy, vocabulary glossary.ProtectedVocabulary, allowRepetitionCleanup bool) ([]chunking.Section, []proposals.AppliedChange, []proposals.SkippedChange, []int, error) {
applied := make([]proposals.AppliedChange, 0)
skipped := make([]proposals.SkippedChange, 0)
appliedBySection := make([]int, len(sections))

View File

@@ -4,6 +4,7 @@ import (
"context"
"strings"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
)
@@ -98,29 +99,25 @@ func (v NoEffectValidator) Validate(_ context.Context, req Request) (Result, err
}
func ValidateProtectedTerms(req Request) (Result, error) {
var vocab ProtectedVocabulary
var vocab glossary.ProtectedVocabulary
if req.GlossaryView != nil {
vocab = req.GlossaryView.Protection
} else {
vocab = NewProtectedVocabulary(req.ProtectedTerms)
vocab = glossary.NewProtectedVocabulary(req.ProtectedTerms)
}
decisions := make([]Decision, 0, len(req.CandidateProposal))
glossaryStage := req.ModuleKey == modulecatalog.KeyGlossary
hasTerms := !vocab.Empty()
if hasTerms {
req.Section = req.sectionLookup()
}
repetitionCleanup := req.ModuleKey == modulecatalog.KeySpokenWord
for _, c := range req.CandidateProposal {
reason := vocab.violationReason(c.OriginalText, c.CorrectedText)
repetitionCleanup := reason != "" && req.ModuleKey == modulecatalog.KeySpokenWord
if vocab.pattern != nil && (glossaryStage || repetitionCleanup) {
req.Section = req.sectionLookup()
var reason string
if hasTerms {
if position, ok := req.Section.Position(c.TargetSegmentID); ok {
segment := &req.Section.segments[position]
preview := req.Preview(c)
if preview.Applicable || (req.IndependentEvidence && preview.SkipReason == proposals.SkipReasonEmptyResultingText) {
if glossaryStage {
// A correction span may sit inside a longer exempt name.
reason = vocab.violationReason(segment.Text, preview.CorrectedSegmentText)
} else if vocab.preservesRepeatedIdentities(segment.Text, preview.CorrectedSegmentText) {
reason = ""
}
if preview.HasResult() {
reason = vocab.Violation(req.Section.segments[position].Text, preview.CorrectedSegmentText, repetitionCleanup)
}
}
}

View File

@@ -11,18 +11,18 @@ import (
// GlossaryView holds the read-only matching data shared by a stage's validators
// and apply-time checks.
type GlossaryView struct {
Protection ProtectedVocabulary
Protection glossary.ProtectedVocabulary
context glossaryContext
}
func PrepareGlossaryView(entries []glossary.Entry, protected []glossary.Term) *GlossaryView {
return &GlossaryView{Protection: NewProtectedVocabulary(protected), context: prepareGlossaryContext(entries)}
return &GlossaryView{Protection: glossary.NewProtectedVocabulary(protected), context: prepareGlossaryContext(entries)}
}
type glossaryContext struct {
entries []glossary.Entry
positions map[glossary.Identity][]int
vocabulary ProtectedVocabulary
vocabulary glossary.ProtectedVocabulary
}
func prepareGlossaryContext(entries []glossary.Entry) glossaryContext {
@@ -31,7 +31,7 @@ func prepareGlossaryContext(entries []glossary.Entry) glossaryContext {
identity := glossary.Identity{Collection: entry.Collection, ID: entry.ID}
positions[identity] = append(positions[identity], i)
}
return glossaryContext{entries: entries, positions: positions, vocabulary: NewProtectedVocabulary(glossary.Vocabulary(entries, nil))}
return glossaryContext{entries: entries, positions: positions, vocabulary: glossary.NewProtectedVocabulary(glossary.Vocabulary(entries, nil))}
}
// validationContext supplies adjacent source segments within the current chunk
@@ -51,12 +51,7 @@ func (view glossaryContext) validationContext(segments []schema.Segment, positio
for _, neighbor := range neighbors {
texts = append(texts, neighbor.Text)
}
mentioned := make(map[glossary.Identity]struct{})
for _, text := range texts {
for _, occurrence := range view.vocabulary.occurrences(text) {
mentioned[occurrence.identity] = struct{}{}
}
}
mentioned := view.vocabulary.MentionedIdentities(texts...)
var matched []int
for identity := range mentioned {
matched = append(matched, view.positions[identity]...)

View File

@@ -1,167 +0,0 @@
package validators
import (
"fmt"
"regexp"
"sort"
"strings"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
)
type protectedTermDef struct {
identities []glossary.Identity
canonical string
}
type protectedOccurrence struct {
text string
identity glossary.Identity
canonical string
}
// ProtectedVocabulary is a deterministic glossary-derived protected term set.
type ProtectedVocabulary struct {
termsByFolded map[string]protectedTermDef
pattern *regexp.Regexp
}
// NewProtectedVocabulary prepares the independently selected terms for matching.
func NewProtectedVocabulary(terms []glossary.Term) ProtectedVocabulary {
termsByFolded := make(map[string]protectedTermDef, len(terms))
alternatives := make([]string, 0, len(terms))
for _, term := range terms {
if !term.Exempt {
termsByFolded[strings.ToLower(term.Spelling)] = protectedTermDef{identities: term.Identities, canonical: term.Spelling}
}
alternatives = append(alternatives, regexp.QuoteMeta(term.Spelling))
}
if len(termsByFolded) == 0 {
return ProtectedVocabulary{}
}
sort.SliceStable(alternatives, func(i, j int) bool { return len(alternatives[i]) > len(alternatives[j]) })
pattern := regexp.MustCompile(`(?i)\b(?:` + strings.Join(alternatives, "|") + `)\b`)
return ProtectedVocabulary{termsByFolded: termsByFolded, pattern: pattern}
}
// AllowsChange checks a proposed segment against the effective protection view.
// Spoken-word cleanup may reduce repeated mentions while preserving each identity.
func (v ProtectedVocabulary) AllowsChange(before, after string, allowRepetitionCleanup bool) bool {
if v.violationReason(before, after) == "" {
return true
}
return allowRepetitionCleanup && v.preservesRepeatedIdentities(before, after)
}
func (v ProtectedVocabulary) violationReason(before, after string) string {
beforeByID := v.occurrencesByIdentity(before)
afterByID := v.occurrencesByIdentity(after)
if reason := validateIdentityPreservation(beforeByID, afterByID); reason != "" {
return reason
}
if reason := validateCapitalizationTransitions(beforeByID, afterByID); reason != "" {
return reason
}
return ""
}
func (v ProtectedVocabulary) occurrencesByIdentity(text string) map[glossary.Identity][]protectedOccurrence {
byID := make(map[glossary.Identity][]protectedOccurrence)
for _, o := range v.occurrences(text) {
byID[o.identity] = append(byID[o.identity], o)
}
return byID
}
func (v ProtectedVocabulary) occurrences(text string) []protectedOccurrence {
if v.pattern == nil {
return nil
}
matches := v.pattern.FindAllStringIndex(text, -1)
if len(matches) == 0 {
return nil
}
out := make([]protectedOccurrence, 0, len(matches))
for _, idx := range matches {
matched := text[idx[0]:idx[1]]
def, ok := v.termsByFolded[strings.ToLower(matched)]
if !ok {
continue
}
for _, identity := range def.identities {
out = append(out, protectedOccurrence{text: matched, identity: identity, canonical: def.canonical})
}
}
return out
}
func validateIdentityPreservation(beforeByID, afterByID map[glossary.Identity][]protectedOccurrence) string {
var selected string
found := false
for identity, beforeItems := range beforeByID {
afterItems := afterByID[identity]
if len(afterItems) >= len(beforeItems) {
continue
}
// Protect identity counts. Within an identity that lost a mention,
// use spelling counts to identify which alias was affected.
remaining := make(map[string]int)
for _, item := range afterItems {
remaining[item.canonical]++
}
for _, item := range beforeItems {
spelling := item.canonical
if remaining[spelling] > 0 {
remaining[spelling]--
continue
}
if !found || spelling < selected {
selected, found = spelling, true
}
}
}
return protectedSpellingMessage(selected, found)
}
func validateCapitalizationTransitions(beforeByID, afterByID map[glossary.Identity][]protectedOccurrence) string {
var selected string
found := false
for identity, afterItems := range afterByID {
beforeItems := beforeByID[identity]
for i, afterItem := range afterItems {
if i >= len(beforeItems) || afterItem.text == beforeItems[i].text || afterItem.text == afterItem.canonical {
continue
}
if !found || afterItem.canonical < selected {
selected, found = afterItem.canonical, true
}
}
}
return protectedSpellingMessage(selected, found)
}
func protectedSpellingMessage(spelling string, found bool) string {
if !found {
return ""
}
return fmt.Sprintf("proposal may alter protected glossary spelling %q", spelling)
}
// preservesRepeatedIdentities permits a spoken-word proposal to reduce repeated
// mentions, but not remove an identity or replace its surviving spelling.
// Editorial and meaning validators still decide whether the cleanup is warranted.
func (v ProtectedVocabulary) preservesRepeatedIdentities(before, after string) bool {
original := v.occurrencesByIdentity(before)
result := v.occurrencesByIdentity(after)
reduced := false
for identity, occurrences := range original {
remaining := result[identity]
if len(remaining) == 0 {
return false
}
if len(remaining) < len(occurrences) {
reduced = true
}
}
return reduced && validateCapitalizationTransitions(original, result) == ""
}

View File

@@ -30,9 +30,11 @@ func TestRecognizedTermsDeduplicateAndIncludePlurals(t *testing.T) {
func TestValidateProtectedTermsAppliesToNonGlossaryModule(t *testing.T) {
req := Request{
ModuleKey: "grammar",
Glossary: []domain.Entry{{ID: "jesters", Name: "Jesters", Category: "faction"}},
ProtectedTerms: domain.Vocabulary([]domain.Entry{{ID: "jesters", Name: "Jesters"}}, nil),
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Jesters"}}},
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
ModuleKey: "grammar",
Glossary: []domain.Entry{{ID: "jesters", Name: "Jesters", Category: "faction"}},
ProtectedTerms: domain.Vocabulary([]domain.Entry{{ID: "jesters", Name: "Jesters"}}, nil),
CandidateProposal: []proposals.EnrichedCorrectionProposal{
{
CorrectionProposal: proposals.CorrectionProposal{
@@ -130,3 +132,32 @@ func TestProtectedNameFindingIdentifiesRemovedAlias(t *testing.T) {
t.Fatalf("finding did not identify the removed alias: %+v", decision)
}
}
func TestProtectionRequiresAUsableFullSegmentPreview(t *testing.T) {
for _, tc := range []struct {
name, source, span, replacement string
target int
rejected bool
}{
{"partial name change", "Cast Mage Armor.", "Mage", "Magus", 1, true},
{"ambiguous span", "Mage Armor and Mage Armor", "Mage Armor", "ward", 1, false},
{"missing span", "Jesters arrived", "Mage Armor", "ward", 1, false},
{"missing target", "Mage Armor", "Mage Armor", "ward", 9, false},
{"empty result", "Mage Armor", "Mage Armor", "", 1, true},
} {
t.Run(tc.name, func(t *testing.T) {
req := Request{
ModuleKey: "grammar", ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
ProtectedTerms: domain.Vocabulary([]domain.Entry{{ID: "mage", Name: "Mage Armor"}}, nil),
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: tc.source}}},
CandidateProposal: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{
TargetSegmentID: tc.target, OriginalText: tc.span, CorrectedText: tc.replacement,
}}},
}
result, err := ValidateProtectedTerms(req)
if err != nil || len(result.Decisions) != 1 || result.Decisions[0].Approved == tc.rejected {
t.Fatalf("protection decision = %+v (%v)", result, err)
}
})
}
}

View File

@@ -140,9 +140,11 @@ func TestNoEffectValidator(t *testing.T) {
func TestValidateProtectedTerms(t *testing.T) {
entries := []domain.Entry{{ID: "openai", Name: "OpenAI", Aliases: []string{"Open AI"}, Category: "brand", Description: "brand"}}
req := Request{
Glossary: entries,
ProtectedTerms: domain.Vocabulary(entries, nil),
ModuleKey: "grammar",
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "OpenAI teh"}}},
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
Glossary: entries,
ProtectedTerms: domain.Vocabulary(entries, nil),
ModuleKey: "grammar",
CandidateProposal: []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "OpenAI", "Open A I"),
mkCandidate(1, 1, "teh", "the"),

View File

@@ -24,7 +24,11 @@ func TestIndependentProtectionForSelectedGlossary(t *testing.T) {
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
result, err := ValidateProtectedTerms(Request{ProtectedTerms: tc.terms, CandidateProposal: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{OriginalText: tc.before, CorrectedText: tc.after}}}})
result, err := ValidateProtectedTerms(Request{
ProtectedTerms: tc.terms, ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: tc.before}}},
CandidateProposal: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: tc.before, CorrectedText: tc.after}}},
})
if err != nil || len(result.Decisions) != 1 || result.Decisions[0].Approved != tc.approved {
t.Fatalf("result=%+v err=%v", result, err)
}
@@ -32,22 +36,10 @@ func TestIndependentProtectionForSelectedGlossary(t *testing.T) {
}
}
func TestSharedAliasRetainsAllIdentities(t *testing.T) {
entries := []glossary.Entry{
{Collection: "a", ID: "first", Name: "First", Aliases: []string{"Shared"}},
{Collection: "b", ID: "second", Name: "Second", Aliases: []string{"Shared"}},
}
vocabulary := NewProtectedVocabulary(glossary.Vocabulary(entries, nil))
identities := vocabulary.occurrencesByIdentity("Shared")
if len(identities) != 2 || len(identities[glossary.Identity{Collection: "a", ID: "first"}]) != 1 || len(identities[glossary.Identity{Collection: "b", ID: "second"}]) != 1 {
t.Fatalf("identities=%v", identities)
}
}
func TestApplicationChecksCurrentTextWithSelectedProtection(t *testing.T) {
active := []glossary.Entry{{Collection: "spells", ID: "spell", Name: "Shield"}}
protected := append(append([]glossary.Entry(nil), active...), glossary.Entry{Collection: "items", ID: "item", Name: "Shield", Aliases: []string{"Aegis"}})
vocabulary := NewProtectedVocabulary(glossary.Vocabulary(protected, active))
vocabulary := glossary.NewProtectedVocabulary(glossary.Vocabulary(protected, active))
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Shield and Aegis"}}}
candidates := []proposals.EnrichedCorrectionProposal{
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "Shield", CorrectedText: "Barrier"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}},
@@ -61,6 +53,32 @@ func TestApplicationChecksCurrentTextWithSelectedProtection(t *testing.T) {
}
}
func TestApplicationRechecksProtectionAfterEarlierEdit(t *testing.T) {
active := []glossary.Entry{{ID: "greater", Name: "Greater Mage Armor"}}
terms := glossary.Vocabulary(append(active, glossary.Entry{ID: "short", Name: "Mage Armor"}), active)
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Cast Greater Mage Armor."}}}
candidates := []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "Greater ", ""),
mkCandidate(1, 1, "Mage Armor", "Magus Armor"),
}
policy := proposals.ReplacementPolicyRequireUnique
validation, err := ValidateProtectedTerms(Request{
ModuleKey: "glossary", WorkingTranscript: transcript, ReplacementPolicy: policy,
ProtectedTerms: terms, CandidateProposal: candidates,
})
if err != nil || len(validation.Decisions) != 2 || !validation.Decisions[0].Approved || !validation.Decisions[1].Approved {
t.Fatalf("initial decisions = %+v (%v)", validation, err)
}
vocabulary := glossary.NewProtectedVocabulary(terms)
result := proposals.ApplyProposals(transcript, candidates, policy, func(before, after string) bool {
return vocabulary.AllowsChange(before, after, false)
})
if result.Transcript.Segments[0].Text != "Cast Mage Armor." || len(result.Applied) != 1 ||
len(result.Skipped) != 1 || result.Skipped[0].ProposalIndex != 1 || result.Skipped[0].SkipReason != proposals.SkipReasonProtectedTerm {
t.Fatalf("stale proposal outcome: %+v", result)
}
}
func TestExemptGlossarySpellingKeepsLongestMatchBoundary(t *testing.T) {
active := []glossary.Entry{
{ID: "greater-mage", Name: "Greater Mage Armor"},
@@ -96,7 +114,7 @@ func TestExemptGlossarySpellingKeepsLongestMatchBoundary(t *testing.T) {
if err != nil || len(decision.Decisions) != 1 || decision.Decisions[0].Approved != tc.allowed {
t.Fatalf("decision=%+v err=%v", decision, err)
}
vocabulary := NewProtectedVocabulary(terms)
vocabulary := glossary.NewProtectedVocabulary(terms)
result := proposals.ApplyProposals(transcript, []proposals.EnrichedCorrectionProposal{candidate}, policy, func(before, after string) bool {
return vocabulary.AllowsChange(before, after, false)
})

View File

@@ -170,6 +170,7 @@ func TestHomophonesProtectedGlossaryTermBehaviorRejectsUnsafeCorrection(t *testi
}}
entries := tinyGlossary()
result, err := protectedValidator.Validate(context.Background(), validators.Request{
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
WorkingTranscript: working,
Glossary: entries,
ProtectedTerms: domain.Vocabulary(entries, nil),

View File

@@ -170,6 +170,7 @@ func TestSpokenWordProtectedGlossaryTermBehaviorRejectsUnsafeCorrection(t *testi
}}
entries := tinyGlossary()
result, err := protectedValidator.Validate(context.Background(), validators.Request{
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
WorkingTranscript: working,
Glossary: entries,
ProtectedTerms: domain.Vocabulary(entries, nil),

View File

@@ -5,6 +5,7 @@ import (
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
@@ -78,9 +79,11 @@ func TestNewRejectsProtectedTermChangeOutsideGlossaryStage(t *testing.T) {
}
req := contracts.ValidationRequest{
ModuleKey: "homophones",
Glossary: tinyGlossary(),
ProtectedTerms: glossary.Vocabulary(tinyGlossary(), nil),
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Jesters"}}},
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
ModuleKey: "homophones",
Glossary: tinyGlossary(),
ProtectedTerms: glossary.Vocabulary(tinyGlossary(), nil),
CandidateProposal: []proposals.EnrichedCorrectionProposal{
proposal(0, "homophones", "Jesters", "Gestures"),
},
@@ -113,9 +116,11 @@ func TestNewGlossaryStageRejectsProtectedTermChange(t *testing.T) {
}
req := contracts.ValidationRequest{
ModuleKey: "glossary",
Glossary: tinyGlossary(),
ProtectedTerms: glossary.Vocabulary(tinyGlossary(), nil),
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Jesters"}}},
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
ModuleKey: "glossary",
Glossary: tinyGlossary(),
ProtectedTerms: glossary.Vocabulary(tinyGlossary(), nil),
CandidateProposal: []proposals.EnrichedCorrectionProposal{
proposal(0, "glossary", "Jesters", "Gestures"),
},
@@ -142,9 +147,11 @@ func TestNewGlossaryStageAllowsProtectedTermInsertion(t *testing.T) {
}
req := contracts.ValidationRequest{
ModuleKey: "glossary",
Glossary: tinyGlossary(),
ProtectedTerms: glossary.Vocabulary(tinyGlossary(), nil),
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "gestures"}}},
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
ModuleKey: "glossary",
Glossary: tinyGlossary(),
ProtectedTerms: glossary.Vocabulary(tinyGlossary(), nil),
CandidateProposal: []proposals.EnrichedCorrectionProposal{
proposal(0, "glossary", "gestures", "Jesters"),
},

View File

@@ -6,6 +6,7 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/validators/editorial_review"
@@ -163,9 +164,11 @@ func TestResolveBuiltInChainUsesGlossaryStageProtectedTermsBehavior(t *testing.T
}
req := contracts.ValidationRequest{
ModuleKey: "glossary",
ProtectedTerms: glossary.Vocabulary([]glossary.Entry{{Collection: "party", ID: "jesters", Name: "Jesters"}}, nil),
Glossary: []glossary.Entry{{Collection: "party", ID: "jesters", Name: "Jesters", Category: "faction", Description: "Protected"}},
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Jesters"}}},
ReplacementPolicy: proposals.ReplacementPolicyReplaceAll,
ModuleKey: "glossary",
ProtectedTerms: glossary.Vocabulary([]glossary.Entry{{Collection: "party", ID: "jesters", Name: "Jesters"}}, nil),
Glossary: []glossary.Entry{{Collection: "party", ID: "jesters", Name: "Jesters", Category: "faction", Description: "Protected"}},
CandidateProposal: []proposals.EnrichedCorrectionProposal{
{
CorrectionProposal: proposals.CorrectionProposal{