Share edit location and check protected names in full segments
This commit is contained in:
@@ -151,12 +151,8 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
for i, raw := range response.Corrections {
|
||||
references = append(references, raw.SegmentReference)
|
||||
// Unknown references remain invalid candidates for the existing domain validators.
|
||||
targetID := 0
|
||||
if req.WorkingTranscript != nil && raw.SegmentReference > 0 && raw.SegmentReference <= len(req.WorkingTranscript.Segments) {
|
||||
targetID = req.WorkingTranscript.Segments[raw.SegmentReference-1].ID
|
||||
}
|
||||
candidate := proposals.CorrectionProposal{
|
||||
TargetSegmentID: targetID,
|
||||
TargetSegmentID: proposals.SegmentIDAtReference(req.WorkingTranscript, raw.SegmentReference),
|
||||
OriginalText: raw.OriginalText,
|
||||
CorrectedText: raw.CorrectedText,
|
||||
}
|
||||
|
||||
@@ -171,13 +171,14 @@ func TestGenerateCandidatesInvalidCorrectionIsPreservedForLaterValidation(t *tes
|
||||
responses: []StructuredCorrectionSet{
|
||||
{
|
||||
Corrections: []StructuredCorrectionProposal{
|
||||
{SegmentReference: 0, OriginalText: "x", CorrectedText: ""},
|
||||
{SegmentReference: 9, OriginalText: "x", CorrectedText: ""},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
req := defaultRequest(t)
|
||||
req.LLMClient = client
|
||||
req.Interactions = llm.NewDiagnosticsWriter(t.TempDir())
|
||||
|
||||
result, err := GenerateCandidates(context.Background(), req)
|
||||
if err != nil {
|
||||
@@ -189,6 +190,20 @@ func TestGenerateCandidatesInvalidCorrectionIsPreservedForLaterValidation(t *tes
|
||||
if result.Corrections[0].TargetSegmentID != 0 {
|
||||
t.Fatalf("unexpected preserved correction: %+v", result.Corrections[0])
|
||||
}
|
||||
raw, err := os.ReadFile(result.Artifacts.ResponsePayloadPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var evidence struct {
|
||||
Decoded StructuredCorrectionSet `json:"decoded"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &evidence); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(result.SegmentReferences) != 1 || result.SegmentReferences[0] != 9 ||
|
||||
len(evidence.Decoded.Corrections) != 1 || evidence.Decoded.Corrections[0].SegmentReference != 9 {
|
||||
t.Fatalf("lost model reference: result=%v artifact=%s", result.SegmentReferences, raw)
|
||||
}
|
||||
if len(result.Operations) != 1 || !result.Operations[0].Succeeded {
|
||||
t.Fatalf("invalid individual correction should remain a usable response: %+v", result.Operations)
|
||||
}
|
||||
|
||||
@@ -170,6 +170,35 @@ func TestApplyProposalsRequireUniqueAmbiguousSkip(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreviewAndApplyHandleOverlappingOccurrences(t *testing.T) {
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "banana"}}}
|
||||
edit := EnrichedCorrectionProposal{CorrectionProposal: CorrectionProposal{
|
||||
TargetSegmentID: 71, OriginalText: "ana", CorrectedText: "X",
|
||||
}}
|
||||
for _, tc := range []struct {
|
||||
policy ReplacementPolicy
|
||||
text string
|
||||
skip ProposalSkipReason
|
||||
}{
|
||||
{ReplacementPolicyRequireUnique, "banana", SkipReasonAmbiguousOriginal},
|
||||
{ReplacementPolicyReplaceAll, "bXna", ""},
|
||||
} {
|
||||
preview := PreviewProposalForSegment(&transcript.Segments[0], edit.CorrectionProposal, tc.policy)
|
||||
result := ApplyProposals(transcript, []EnrichedCorrectionProposal{edit}, tc.policy, nil)
|
||||
if result.Transcript.Segments[0].Text != tc.text {
|
||||
t.Fatalf("%s produced %q", tc.policy, result.Transcript.Segments[0].Text)
|
||||
}
|
||||
if tc.skip != "" {
|
||||
if preview.HasResult() || preview.SkipReason != tc.skip || len(result.Skipped) != 1 || result.Skipped[0].SkipReason != tc.skip {
|
||||
t.Fatalf("%s preview %+v and apply %+v disagree with expected skip", tc.policy, preview, result)
|
||||
}
|
||||
} else if !preview.Applicable || preview.CorrectedSegmentText != tc.text ||
|
||||
len(result.Applied) != 1 || result.Applied[0].ReplacementCount != 1 {
|
||||
t.Fatalf("%s preview %+v and apply %+v disagree with expected edit", tc.policy, preview, result)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyProposalsReplaceAllMultipleReplacements(t *testing.T) {
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "uh uh uh"}}}
|
||||
proposals := []EnrichedCorrectionProposal{
|
||||
|
||||
@@ -48,42 +48,34 @@ func PreviewProposalForSegment(segment *schema.Segment, proposal CorrectionPropo
|
||||
return SegmentPreviewResult{SkipReason: SkipReasonMissingSegment}
|
||||
}
|
||||
|
||||
matchCount := strings.Count(segment.Text, proposal.OriginalText)
|
||||
if matchCount == 0 {
|
||||
first := strings.Index(segment.Text, proposal.OriginalText)
|
||||
if first < 0 {
|
||||
return SegmentPreviewResult{SkipReason: SkipReasonMissingOriginalText}
|
||||
}
|
||||
|
||||
if proposal.OriginalText == proposal.CorrectedText {
|
||||
return SegmentPreviewResult{SkipReason: SkipReasonNoEffect}
|
||||
}
|
||||
|
||||
switch policy {
|
||||
case ReplacementPolicyRequireUnique:
|
||||
if matchCount != 1 {
|
||||
count := 1
|
||||
var corrected string
|
||||
if policy == ReplacementPolicyRequireUnique {
|
||||
// Search after the first starting offset, including overlapping matches.
|
||||
if strings.Contains(segment.Text[first+1:], proposal.OriginalText) {
|
||||
return SegmentPreviewResult{SkipReason: SkipReasonAmbiguousOriginal}
|
||||
}
|
||||
|
||||
corrected := strings.Replace(segment.Text, proposal.OriginalText, proposal.CorrectedText, 1)
|
||||
if strings.TrimSpace(corrected) == "" {
|
||||
return SegmentPreviewResult{SkipReason: SkipReasonEmptyResultingText, CorrectedSegmentText: corrected}
|
||||
}
|
||||
return SegmentPreviewResult{
|
||||
Applicable: true,
|
||||
CorrectedSegmentText: corrected,
|
||||
ReplacementCount: 1,
|
||||
}
|
||||
|
||||
case ReplacementPolicyReplaceAll:
|
||||
corrected := strings.ReplaceAll(segment.Text, proposal.OriginalText, proposal.CorrectedText)
|
||||
if strings.TrimSpace(corrected) == "" {
|
||||
return SegmentPreviewResult{SkipReason: SkipReasonEmptyResultingText, CorrectedSegmentText: corrected}
|
||||
}
|
||||
return SegmentPreviewResult{
|
||||
Applicable: true,
|
||||
CorrectedSegmentText: corrected,
|
||||
ReplacementCount: matchCount,
|
||||
}
|
||||
corrected = strings.Replace(segment.Text, proposal.OriginalText, proposal.CorrectedText, 1)
|
||||
} else {
|
||||
count = strings.Count(segment.Text, proposal.OriginalText)
|
||||
corrected = strings.ReplaceAll(segment.Text, proposal.OriginalText, proposal.CorrectedText)
|
||||
}
|
||||
|
||||
return SegmentPreviewResult{SkipReason: SkipReasonInvalidProposal}
|
||||
if strings.TrimSpace(corrected) == "" {
|
||||
return SegmentPreviewResult{SkipReason: SkipReasonEmptyResultingText, CorrectedSegmentText: corrected}
|
||||
}
|
||||
return SegmentPreviewResult{Applicable: true, CorrectedSegmentText: corrected, ReplacementCount: count}
|
||||
}
|
||||
|
||||
// HasResult indicates that the replacement was located and yielded a preview,
|
||||
// including when a separate blank-result rule makes the edit inapplicable.
|
||||
func (r SegmentPreviewResult) HasResult() bool {
|
||||
return r.Applicable || r.SkipReason == SkipReasonEmptyResultingText
|
||||
}
|
||||
|
||||
@@ -2,6 +2,8 @@ package proposals
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
)
|
||||
|
||||
// CorrectionProposal is the base proposal emitted by correction modules.
|
||||
@@ -39,3 +41,12 @@ func (p CorrectionProposal) Validate() error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// SegmentIDAtReference maps a one-based model reference in the supplied
|
||||
// section to the transcript's segment ID. Zero represents an invalid reference.
|
||||
func SegmentIDAtReference(section *schema.Transcript, reference int) int {
|
||||
if section == nil || reference <= 0 || reference > len(section.Segments) {
|
||||
return 0
|
||||
}
|
||||
return section.Segments[reference-1].ID
|
||||
}
|
||||
|
||||
@@ -204,12 +204,8 @@ func repairCandidates(ctx context.Context, input validateSectionCandidatesInput,
|
||||
}
|
||||
revision := response.Decision.Correction
|
||||
localRef = revision.SegmentReference
|
||||
target := 0
|
||||
if input.WorkingTranscript != nil && localRef > 0 && localRef <= len(input.WorkingTranscript.Segments) {
|
||||
target = input.WorkingTranscript.Segments[localRef-1].ID
|
||||
}
|
||||
current = original
|
||||
current.CorrectionProposal = proposals.CorrectionProposal{TargetSegmentID: target, OriginalText: revision.OriginalText, CorrectedText: revision.CorrectedText}
|
||||
current.CorrectionProposal = proposals.CorrectionProposal{TargetSegmentID: proposals.SegmentIDAtReference(input.WorkingTranscript, localRef), OriginalText: revision.OriginalText, CorrectedText: revision.CorrectedText}
|
||||
if debug {
|
||||
branch.Final = current
|
||||
}
|
||||
|
||||
@@ -130,7 +130,9 @@ func TestRepairRequestRefreshesPreviewAfterRetarget(t *testing.T) {
|
||||
`{"proposal_ref":1,"decision":{"action":"revise","correction":{"id":2,"original_text":"uh ","corrected_text":""}}}`,
|
||||
`{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`,
|
||||
}}
|
||||
var validatedIDs []int
|
||||
reject := fakeValidator{name: "editorial_review", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
validatedIDs = append(validatedIDs, req.CandidateProposal[0].TargetSegmentID)
|
||||
return validators.Result{Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Disposition: validators.DispositionRejected, ReasonCode: "llm_rejected", Message: "The context does not support this edit."}}}, nil
|
||||
}}
|
||||
input := validateSectionCandidatesInput{
|
||||
@@ -149,6 +151,9 @@ func TestRepairRequestRefreshesPreviewAfterRetarget(t *testing.T) {
|
||||
if err != nil || len(client.calls) != 2 || len(branches) != 1 || branches[0].Outcome != "withdrawn" {
|
||||
t.Fatalf("repair: branches=%+v calls=%d err=%v", branches, len(client.calls), err)
|
||||
}
|
||||
if len(validatedIDs) != 2 || validatedIDs[0] != 81 || validatedIDs[1] != 104 {
|
||||
t.Fatalf("validated segment IDs: %v", validatedIDs)
|
||||
}
|
||||
for i, want := range []struct{ before, after string }{{"cat", "dog"}, {"uh hello", "hello"}} {
|
||||
var payload proposal_generation.RepairRequest
|
||||
if err := json.Unmarshal([]byte(client.calls[i].Variables["RepairRequestJSON"]), &payload); err != nil {
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||
@@ -290,7 +291,7 @@ func transcriptFromSections(sections []chunking.Section) *schema.Transcript {
|
||||
|
||||
// applyWithinSections leaves section membership and initial metadata unchanged.
|
||||
// Approved proposals are applied only to their identified section.
|
||||
func applyWithinSections(sections []chunking.Section, approved []proposals.EnrichedCorrectionProposal, policy proposals.ReplacementPolicy, vocabulary validators.ProtectedVocabulary, allowRepetitionCleanup bool) ([]chunking.Section, []proposals.AppliedChange, []proposals.SkippedChange, []int, error) {
|
||||
func applyWithinSections(sections []chunking.Section, approved []proposals.EnrichedCorrectionProposal, policy proposals.ReplacementPolicy, vocabulary glossary.ProtectedVocabulary, allowRepetitionCleanup bool) ([]chunking.Section, []proposals.AppliedChange, []proposals.SkippedChange, []int, error) {
|
||||
applied := make([]proposals.AppliedChange, 0)
|
||||
skipped := make([]proposals.SkippedChange, 0)
|
||||
appliedBySection := make([]int, len(sections))
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||
)
|
||||
@@ -98,29 +99,25 @@ func (v NoEffectValidator) Validate(_ context.Context, req Request) (Result, err
|
||||
}
|
||||
|
||||
func ValidateProtectedTerms(req Request) (Result, error) {
|
||||
var vocab ProtectedVocabulary
|
||||
var vocab glossary.ProtectedVocabulary
|
||||
if req.GlossaryView != nil {
|
||||
vocab = req.GlossaryView.Protection
|
||||
} else {
|
||||
vocab = NewProtectedVocabulary(req.ProtectedTerms)
|
||||
vocab = glossary.NewProtectedVocabulary(req.ProtectedTerms)
|
||||
}
|
||||
decisions := make([]Decision, 0, len(req.CandidateProposal))
|
||||
glossaryStage := req.ModuleKey == modulecatalog.KeyGlossary
|
||||
hasTerms := !vocab.Empty()
|
||||
if hasTerms {
|
||||
req.Section = req.sectionLookup()
|
||||
}
|
||||
repetitionCleanup := req.ModuleKey == modulecatalog.KeySpokenWord
|
||||
for _, c := range req.CandidateProposal {
|
||||
reason := vocab.violationReason(c.OriginalText, c.CorrectedText)
|
||||
repetitionCleanup := reason != "" && req.ModuleKey == modulecatalog.KeySpokenWord
|
||||
if vocab.pattern != nil && (glossaryStage || repetitionCleanup) {
|
||||
req.Section = req.sectionLookup()
|
||||
var reason string
|
||||
if hasTerms {
|
||||
if position, ok := req.Section.Position(c.TargetSegmentID); ok {
|
||||
segment := &req.Section.segments[position]
|
||||
preview := req.Preview(c)
|
||||
if preview.Applicable || (req.IndependentEvidence && preview.SkipReason == proposals.SkipReasonEmptyResultingText) {
|
||||
if glossaryStage {
|
||||
// A correction span may sit inside a longer exempt name.
|
||||
reason = vocab.violationReason(segment.Text, preview.CorrectedSegmentText)
|
||||
} else if vocab.preservesRepeatedIdentities(segment.Text, preview.CorrectedSegmentText) {
|
||||
reason = ""
|
||||
}
|
||||
if preview.HasResult() {
|
||||
reason = vocab.Violation(req.Section.segments[position].Text, preview.CorrectedSegmentText, repetitionCleanup)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,18 +11,18 @@ import (
|
||||
// GlossaryView holds the read-only matching data shared by a stage's validators
|
||||
// and apply-time checks.
|
||||
type GlossaryView struct {
|
||||
Protection ProtectedVocabulary
|
||||
Protection glossary.ProtectedVocabulary
|
||||
context glossaryContext
|
||||
}
|
||||
|
||||
func PrepareGlossaryView(entries []glossary.Entry, protected []glossary.Term) *GlossaryView {
|
||||
return &GlossaryView{Protection: NewProtectedVocabulary(protected), context: prepareGlossaryContext(entries)}
|
||||
return &GlossaryView{Protection: glossary.NewProtectedVocabulary(protected), context: prepareGlossaryContext(entries)}
|
||||
}
|
||||
|
||||
type glossaryContext struct {
|
||||
entries []glossary.Entry
|
||||
positions map[glossary.Identity][]int
|
||||
vocabulary ProtectedVocabulary
|
||||
vocabulary glossary.ProtectedVocabulary
|
||||
}
|
||||
|
||||
func prepareGlossaryContext(entries []glossary.Entry) glossaryContext {
|
||||
@@ -31,7 +31,7 @@ func prepareGlossaryContext(entries []glossary.Entry) glossaryContext {
|
||||
identity := glossary.Identity{Collection: entry.Collection, ID: entry.ID}
|
||||
positions[identity] = append(positions[identity], i)
|
||||
}
|
||||
return glossaryContext{entries: entries, positions: positions, vocabulary: NewProtectedVocabulary(glossary.Vocabulary(entries, nil))}
|
||||
return glossaryContext{entries: entries, positions: positions, vocabulary: glossary.NewProtectedVocabulary(glossary.Vocabulary(entries, nil))}
|
||||
}
|
||||
|
||||
// validationContext supplies adjacent source segments within the current chunk
|
||||
@@ -51,12 +51,7 @@ func (view glossaryContext) validationContext(segments []schema.Segment, positio
|
||||
for _, neighbor := range neighbors {
|
||||
texts = append(texts, neighbor.Text)
|
||||
}
|
||||
mentioned := make(map[glossary.Identity]struct{})
|
||||
for _, text := range texts {
|
||||
for _, occurrence := range view.vocabulary.occurrences(text) {
|
||||
mentioned[occurrence.identity] = struct{}{}
|
||||
}
|
||||
}
|
||||
mentioned := view.vocabulary.MentionedIdentities(texts...)
|
||||
var matched []int
|
||||
for identity := range mentioned {
|
||||
matched = append(matched, view.positions[identity]...)
|
||||
|
||||
@@ -1,167 +0,0 @@
|
||||
package validators
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
|
||||
)
|
||||
|
||||
type protectedTermDef struct {
|
||||
identities []glossary.Identity
|
||||
canonical string
|
||||
}
|
||||
|
||||
type protectedOccurrence struct {
|
||||
text string
|
||||
identity glossary.Identity
|
||||
canonical string
|
||||
}
|
||||
|
||||
// ProtectedVocabulary is a deterministic glossary-derived protected term set.
|
||||
type ProtectedVocabulary struct {
|
||||
termsByFolded map[string]protectedTermDef
|
||||
pattern *regexp.Regexp
|
||||
}
|
||||
|
||||
// NewProtectedVocabulary prepares the independently selected terms for matching.
|
||||
func NewProtectedVocabulary(terms []glossary.Term) ProtectedVocabulary {
|
||||
termsByFolded := make(map[string]protectedTermDef, len(terms))
|
||||
alternatives := make([]string, 0, len(terms))
|
||||
for _, term := range terms {
|
||||
if !term.Exempt {
|
||||
termsByFolded[strings.ToLower(term.Spelling)] = protectedTermDef{identities: term.Identities, canonical: term.Spelling}
|
||||
}
|
||||
alternatives = append(alternatives, regexp.QuoteMeta(term.Spelling))
|
||||
}
|
||||
if len(termsByFolded) == 0 {
|
||||
return ProtectedVocabulary{}
|
||||
}
|
||||
sort.SliceStable(alternatives, func(i, j int) bool { return len(alternatives[i]) > len(alternatives[j]) })
|
||||
pattern := regexp.MustCompile(`(?i)\b(?:` + strings.Join(alternatives, "|") + `)\b`)
|
||||
return ProtectedVocabulary{termsByFolded: termsByFolded, pattern: pattern}
|
||||
}
|
||||
|
||||
// AllowsChange checks a proposed segment against the effective protection view.
|
||||
// Spoken-word cleanup may reduce repeated mentions while preserving each identity.
|
||||
func (v ProtectedVocabulary) AllowsChange(before, after string, allowRepetitionCleanup bool) bool {
|
||||
if v.violationReason(before, after) == "" {
|
||||
return true
|
||||
}
|
||||
return allowRepetitionCleanup && v.preservesRepeatedIdentities(before, after)
|
||||
}
|
||||
|
||||
func (v ProtectedVocabulary) violationReason(before, after string) string {
|
||||
beforeByID := v.occurrencesByIdentity(before)
|
||||
afterByID := v.occurrencesByIdentity(after)
|
||||
if reason := validateIdentityPreservation(beforeByID, afterByID); reason != "" {
|
||||
return reason
|
||||
}
|
||||
if reason := validateCapitalizationTransitions(beforeByID, afterByID); reason != "" {
|
||||
return reason
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (v ProtectedVocabulary) occurrencesByIdentity(text string) map[glossary.Identity][]protectedOccurrence {
|
||||
byID := make(map[glossary.Identity][]protectedOccurrence)
|
||||
for _, o := range v.occurrences(text) {
|
||||
byID[o.identity] = append(byID[o.identity], o)
|
||||
}
|
||||
return byID
|
||||
}
|
||||
|
||||
func (v ProtectedVocabulary) occurrences(text string) []protectedOccurrence {
|
||||
if v.pattern == nil {
|
||||
return nil
|
||||
}
|
||||
matches := v.pattern.FindAllStringIndex(text, -1)
|
||||
if len(matches) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]protectedOccurrence, 0, len(matches))
|
||||
for _, idx := range matches {
|
||||
matched := text[idx[0]:idx[1]]
|
||||
def, ok := v.termsByFolded[strings.ToLower(matched)]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for _, identity := range def.identities {
|
||||
out = append(out, protectedOccurrence{text: matched, identity: identity, canonical: def.canonical})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func validateIdentityPreservation(beforeByID, afterByID map[glossary.Identity][]protectedOccurrence) string {
|
||||
var selected string
|
||||
found := false
|
||||
for identity, beforeItems := range beforeByID {
|
||||
afterItems := afterByID[identity]
|
||||
if len(afterItems) >= len(beforeItems) {
|
||||
continue
|
||||
}
|
||||
// Protect identity counts. Within an identity that lost a mention,
|
||||
// use spelling counts to identify which alias was affected.
|
||||
remaining := make(map[string]int)
|
||||
for _, item := range afterItems {
|
||||
remaining[item.canonical]++
|
||||
}
|
||||
for _, item := range beforeItems {
|
||||
spelling := item.canonical
|
||||
if remaining[spelling] > 0 {
|
||||
remaining[spelling]--
|
||||
continue
|
||||
}
|
||||
if !found || spelling < selected {
|
||||
selected, found = spelling, true
|
||||
}
|
||||
}
|
||||
}
|
||||
return protectedSpellingMessage(selected, found)
|
||||
}
|
||||
|
||||
func validateCapitalizationTransitions(beforeByID, afterByID map[glossary.Identity][]protectedOccurrence) string {
|
||||
var selected string
|
||||
found := false
|
||||
for identity, afterItems := range afterByID {
|
||||
beforeItems := beforeByID[identity]
|
||||
for i, afterItem := range afterItems {
|
||||
if i >= len(beforeItems) || afterItem.text == beforeItems[i].text || afterItem.text == afterItem.canonical {
|
||||
continue
|
||||
}
|
||||
if !found || afterItem.canonical < selected {
|
||||
selected, found = afterItem.canonical, true
|
||||
}
|
||||
}
|
||||
}
|
||||
return protectedSpellingMessage(selected, found)
|
||||
}
|
||||
|
||||
func protectedSpellingMessage(spelling string, found bool) string {
|
||||
if !found {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("proposal may alter protected glossary spelling %q", spelling)
|
||||
}
|
||||
|
||||
// preservesRepeatedIdentities permits a spoken-word proposal to reduce repeated
|
||||
// mentions, but not remove an identity or replace its surviving spelling.
|
||||
// Editorial and meaning validators still decide whether the cleanup is warranted.
|
||||
func (v ProtectedVocabulary) preservesRepeatedIdentities(before, after string) bool {
|
||||
original := v.occurrencesByIdentity(before)
|
||||
result := v.occurrencesByIdentity(after)
|
||||
reduced := false
|
||||
for identity, occurrences := range original {
|
||||
remaining := result[identity]
|
||||
if len(remaining) == 0 {
|
||||
return false
|
||||
}
|
||||
if len(remaining) < len(occurrences) {
|
||||
reduced = true
|
||||
}
|
||||
}
|
||||
return reduced && validateCapitalizationTransitions(original, result) == ""
|
||||
}
|
||||
@@ -30,9 +30,11 @@ func TestRecognizedTermsDeduplicateAndIncludePlurals(t *testing.T) {
|
||||
|
||||
func TestValidateProtectedTermsAppliesToNonGlossaryModule(t *testing.T) {
|
||||
req := Request{
|
||||
ModuleKey: "grammar",
|
||||
Glossary: []domain.Entry{{ID: "jesters", Name: "Jesters", Category: "faction"}},
|
||||
ProtectedTerms: domain.Vocabulary([]domain.Entry{{ID: "jesters", Name: "Jesters"}}, nil),
|
||||
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Jesters"}}},
|
||||
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
||||
ModuleKey: "grammar",
|
||||
Glossary: []domain.Entry{{ID: "jesters", Name: "Jesters", Category: "faction"}},
|
||||
ProtectedTerms: domain.Vocabulary([]domain.Entry{{ID: "jesters", Name: "Jesters"}}, nil),
|
||||
CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
||||
{
|
||||
CorrectionProposal: proposals.CorrectionProposal{
|
||||
@@ -130,3 +132,32 @@ func TestProtectedNameFindingIdentifiesRemovedAlias(t *testing.T) {
|
||||
t.Fatalf("finding did not identify the removed alias: %+v", decision)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProtectionRequiresAUsableFullSegmentPreview(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, source, span, replacement string
|
||||
target int
|
||||
rejected bool
|
||||
}{
|
||||
{"partial name change", "Cast Mage Armor.", "Mage", "Magus", 1, true},
|
||||
{"ambiguous span", "Mage Armor and Mage Armor", "Mage Armor", "ward", 1, false},
|
||||
{"missing span", "Jesters arrived", "Mage Armor", "ward", 1, false},
|
||||
{"missing target", "Mage Armor", "Mage Armor", "ward", 9, false},
|
||||
{"empty result", "Mage Armor", "Mage Armor", "", 1, true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
req := Request{
|
||||
ModuleKey: "grammar", ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
||||
ProtectedTerms: domain.Vocabulary([]domain.Entry{{ID: "mage", Name: "Mage Armor"}}, nil),
|
||||
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: tc.source}}},
|
||||
CandidateProposal: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{
|
||||
TargetSegmentID: tc.target, OriginalText: tc.span, CorrectedText: tc.replacement,
|
||||
}}},
|
||||
}
|
||||
result, err := ValidateProtectedTerms(req)
|
||||
if err != nil || len(result.Decisions) != 1 || result.Decisions[0].Approved == tc.rejected {
|
||||
t.Fatalf("protection decision = %+v (%v)", result, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -140,9 +140,11 @@ func TestNoEffectValidator(t *testing.T) {
|
||||
func TestValidateProtectedTerms(t *testing.T) {
|
||||
entries := []domain.Entry{{ID: "openai", Name: "OpenAI", Aliases: []string{"Open AI"}, Category: "brand", Description: "brand"}}
|
||||
req := Request{
|
||||
Glossary: entries,
|
||||
ProtectedTerms: domain.Vocabulary(entries, nil),
|
||||
ModuleKey: "grammar",
|
||||
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "OpenAI teh"}}},
|
||||
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
||||
Glossary: entries,
|
||||
ProtectedTerms: domain.Vocabulary(entries, nil),
|
||||
ModuleKey: "grammar",
|
||||
CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
||||
mkCandidate(0, 1, "OpenAI", "Open A I"),
|
||||
mkCandidate(1, 1, "teh", "the"),
|
||||
|
||||
@@ -24,7 +24,11 @@ func TestIndependentProtectionForSelectedGlossary(t *testing.T) {
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
result, err := ValidateProtectedTerms(Request{ProtectedTerms: tc.terms, CandidateProposal: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{OriginalText: tc.before, CorrectedText: tc.after}}}})
|
||||
result, err := ValidateProtectedTerms(Request{
|
||||
ProtectedTerms: tc.terms, ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
||||
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: tc.before}}},
|
||||
CandidateProposal: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: tc.before, CorrectedText: tc.after}}},
|
||||
})
|
||||
if err != nil || len(result.Decisions) != 1 || result.Decisions[0].Approved != tc.approved {
|
||||
t.Fatalf("result=%+v err=%v", result, err)
|
||||
}
|
||||
@@ -32,22 +36,10 @@ func TestIndependentProtectionForSelectedGlossary(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSharedAliasRetainsAllIdentities(t *testing.T) {
|
||||
entries := []glossary.Entry{
|
||||
{Collection: "a", ID: "first", Name: "First", Aliases: []string{"Shared"}},
|
||||
{Collection: "b", ID: "second", Name: "Second", Aliases: []string{"Shared"}},
|
||||
}
|
||||
vocabulary := NewProtectedVocabulary(glossary.Vocabulary(entries, nil))
|
||||
identities := vocabulary.occurrencesByIdentity("Shared")
|
||||
if len(identities) != 2 || len(identities[glossary.Identity{Collection: "a", ID: "first"}]) != 1 || len(identities[glossary.Identity{Collection: "b", ID: "second"}]) != 1 {
|
||||
t.Fatalf("identities=%v", identities)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplicationChecksCurrentTextWithSelectedProtection(t *testing.T) {
|
||||
active := []glossary.Entry{{Collection: "spells", ID: "spell", Name: "Shield"}}
|
||||
protected := append(append([]glossary.Entry(nil), active...), glossary.Entry{Collection: "items", ID: "item", Name: "Shield", Aliases: []string{"Aegis"}})
|
||||
vocabulary := NewProtectedVocabulary(glossary.Vocabulary(protected, active))
|
||||
vocabulary := glossary.NewProtectedVocabulary(glossary.Vocabulary(protected, active))
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Shield and Aegis"}}}
|
||||
candidates := []proposals.EnrichedCorrectionProposal{
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "Shield", CorrectedText: "Barrier"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}},
|
||||
@@ -61,6 +53,32 @@ func TestApplicationChecksCurrentTextWithSelectedProtection(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplicationRechecksProtectionAfterEarlierEdit(t *testing.T) {
|
||||
active := []glossary.Entry{{ID: "greater", Name: "Greater Mage Armor"}}
|
||||
terms := glossary.Vocabulary(append(active, glossary.Entry{ID: "short", Name: "Mage Armor"}), active)
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Cast Greater Mage Armor."}}}
|
||||
candidates := []proposals.EnrichedCorrectionProposal{
|
||||
mkCandidate(0, 1, "Greater ", ""),
|
||||
mkCandidate(1, 1, "Mage Armor", "Magus Armor"),
|
||||
}
|
||||
policy := proposals.ReplacementPolicyRequireUnique
|
||||
validation, err := ValidateProtectedTerms(Request{
|
||||
ModuleKey: "glossary", WorkingTranscript: transcript, ReplacementPolicy: policy,
|
||||
ProtectedTerms: terms, CandidateProposal: candidates,
|
||||
})
|
||||
if err != nil || len(validation.Decisions) != 2 || !validation.Decisions[0].Approved || !validation.Decisions[1].Approved {
|
||||
t.Fatalf("initial decisions = %+v (%v)", validation, err)
|
||||
}
|
||||
vocabulary := glossary.NewProtectedVocabulary(terms)
|
||||
result := proposals.ApplyProposals(transcript, candidates, policy, func(before, after string) bool {
|
||||
return vocabulary.AllowsChange(before, after, false)
|
||||
})
|
||||
if result.Transcript.Segments[0].Text != "Cast Mage Armor." || len(result.Applied) != 1 ||
|
||||
len(result.Skipped) != 1 || result.Skipped[0].ProposalIndex != 1 || result.Skipped[0].SkipReason != proposals.SkipReasonProtectedTerm {
|
||||
t.Fatalf("stale proposal outcome: %+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExemptGlossarySpellingKeepsLongestMatchBoundary(t *testing.T) {
|
||||
active := []glossary.Entry{
|
||||
{ID: "greater-mage", Name: "Greater Mage Armor"},
|
||||
@@ -96,7 +114,7 @@ func TestExemptGlossarySpellingKeepsLongestMatchBoundary(t *testing.T) {
|
||||
if err != nil || len(decision.Decisions) != 1 || decision.Decisions[0].Approved != tc.allowed {
|
||||
t.Fatalf("decision=%+v err=%v", decision, err)
|
||||
}
|
||||
vocabulary := NewProtectedVocabulary(terms)
|
||||
vocabulary := glossary.NewProtectedVocabulary(terms)
|
||||
result := proposals.ApplyProposals(transcript, []proposals.EnrichedCorrectionProposal{candidate}, policy, func(before, after string) bool {
|
||||
return vocabulary.AllowsChange(before, after, false)
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user