Derive independent glossary context and protection
This commit is contained in:
@@ -471,6 +471,12 @@ validation-profile fallback cases and meaningful explicit-value distinctions.
|
|||||||
|
|
||||||
## Stage 11: Derive stage vocabulary and adapt its consumers
|
## Stage 11: Derive stage vocabulary and adapt its consumers
|
||||||
|
|
||||||
|
**Completed.** Rich context and independent, identity-aware protection views
|
||||||
|
are available to prompt, validation, and application consumers. The ordered
|
||||||
|
runner must supply those views and the application check in the next assignment;
|
||||||
|
older caller tests remain to be reconciled during the assembled cutover. Focused
|
||||||
|
projection, validator, and application checks passed.
|
||||||
|
|
||||||
Own the glossary-domain projection and the consumers in `internal/modules`,
|
Own the glossary-domain projection and the consumers in `internal/modules`,
|
||||||
`internal/validators`, and `internal/framework/proposals`. Use the
|
`internal/validators`, and `internal/framework/proposals`. Use the
|
||||||
[derived views](glossary-stages.md#derived-runtime-representations--resolved)
|
[derived views](glossary-stages.md#derived-runtime-representations--resolved)
|
||||||
|
|||||||
39
internal/core/glossary/selection.go
Normal file
39
internal/core/glossary/selection.go
Normal file
@@ -0,0 +1,39 @@
|
|||||||
|
// Package glossary derives consumer views from resolved configuration.
|
||||||
|
package glossary
|
||||||
|
|
||||||
|
import "gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||||
|
|
||||||
|
// Entry is the known glossary content and its collection-scoped identity.
|
||||||
|
type Entry struct {
|
||||||
|
Collection string `json:"-"`
|
||||||
|
ID string `json:"id"`
|
||||||
|
Name string `json:"name"`
|
||||||
|
Category string `json:"category"`
|
||||||
|
Description string `json:"description,omitempty"`
|
||||||
|
Aliases []string `json:"aliases,omitempty"`
|
||||||
|
Plural *string `json:"plural,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type identity struct{ collection, id string }
|
||||||
|
|
||||||
|
// Select retains the first occurrence of each selected collection entry.
|
||||||
|
func Select(collections map[string]config.GlossaryCollection, names []string) []Entry {
|
||||||
|
seen := make(map[identity]struct{})
|
||||||
|
var selected []Entry
|
||||||
|
for _, name := range names {
|
||||||
|
for _, item := range collections[name].Entries {
|
||||||
|
key := identity{collection: name, id: item.ID}
|
||||||
|
if _, exists := seen[key]; exists {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[key] = struct{}{}
|
||||||
|
entry := Entry{Collection: name, ID: item.ID, Name: item.Name, Category: item.Category, Description: item.Description, Aliases: append([]string(nil), item.Aliases...)}
|
||||||
|
if item.Plural != nil {
|
||||||
|
plural := *item.Plural
|
||||||
|
entry.Plural = &plural
|
||||||
|
}
|
||||||
|
selected = append(selected, entry)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return selected
|
||||||
|
}
|
||||||
33
internal/core/glossary/selection_test.go
Normal file
33
internal/core/glossary/selection_test.go
Normal file
@@ -0,0 +1,33 @@
|
|||||||
|
package glossary
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestSelectionRetainsDistinctIdentitiesAndDoesNotExposeExtensions(t *testing.T) {
|
||||||
|
plural := "Shields"
|
||||||
|
collections := map[string]config.GlossaryCollection{
|
||||||
|
"spells": {Entries: []config.GlossaryEntry{{ID: "shield", Name: "Shield", Category: "spell", Aliases: []string{"Ward"}, Plural: &plural, Extra: map[string]any{"private": "do not forward"}}}},
|
||||||
|
"items": {Entries: []config.GlossaryEntry{{ID: "shield", Name: "Shield", Category: "item"}}},
|
||||||
|
}
|
||||||
|
result := Select(collections, []string{"spells", "items", "spells"})
|
||||||
|
if len(result) != 2 || result[0].Collection != "spells" || result[1].Collection != "items" {
|
||||||
|
t.Fatalf("entries=%+v", result)
|
||||||
|
}
|
||||||
|
raw, err := json.Marshal(result)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if strings.Contains(string(raw), "do not forward") {
|
||||||
|
t.Fatal("extension leaked into prompt projection")
|
||||||
|
}
|
||||||
|
result[0].Aliases[0] = "changed"
|
||||||
|
*result[0].Plural = "changed"
|
||||||
|
if collections["spells"].Entries[0].Aliases[0] != "Ward" || *collections["spells"].Entries[0].Plural != "Shields" {
|
||||||
|
t.Fatal("projection changed configuration")
|
||||||
|
}
|
||||||
|
}
|
||||||
71
internal/core/glossary/vocabulary.go
Normal file
71
internal/core/glossary/vocabulary.go
Normal file
@@ -0,0 +1,71 @@
|
|||||||
|
package glossary
|
||||||
|
|
||||||
|
import "strings"
|
||||||
|
|
||||||
|
// Identity identifies an entry within its configured collection.
|
||||||
|
type Identity struct {
|
||||||
|
Collection string
|
||||||
|
ID string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Term retains every entry that contributes a recognized spelling.
|
||||||
|
type Term struct {
|
||||||
|
Spelling string
|
||||||
|
Identities []Identity
|
||||||
|
}
|
||||||
|
|
||||||
|
// Vocabulary derives recognized spellings in selection order. Excluded entries
|
||||||
|
// remove matching spellings regardless of which protected entries supplied them.
|
||||||
|
func Vocabulary(entries, excluded []Entry) []Term {
|
||||||
|
exclusions := make(map[string]struct{})
|
||||||
|
for _, entry := range excluded {
|
||||||
|
for _, spelling := range spellings(entry) {
|
||||||
|
exclusions[strings.ToLower(spelling)] = struct{}{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
indices := make(map[string]int)
|
||||||
|
var terms []Term
|
||||||
|
for _, entry := range entries {
|
||||||
|
id := Identity{Collection: entry.Collection, ID: entry.ID}
|
||||||
|
for _, spelling := range spellings(entry) {
|
||||||
|
key := strings.ToLower(spelling)
|
||||||
|
if _, excluded := exclusions[key]; excluded {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
index, exists := indices[key]
|
||||||
|
if !exists {
|
||||||
|
index = len(terms)
|
||||||
|
indices[key] = index
|
||||||
|
terms = append(terms, Term{Spelling: spelling})
|
||||||
|
}
|
||||||
|
identities := terms[index].Identities
|
||||||
|
found := false
|
||||||
|
for _, existing := range identities {
|
||||||
|
if existing == id {
|
||||||
|
found = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
terms[index].Identities = append(identities, id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return terms
|
||||||
|
}
|
||||||
|
|
||||||
|
func spellings(entry Entry) []string {
|
||||||
|
var result []string
|
||||||
|
for _, value := range append([]string{entry.Name}, entry.Aliases...) {
|
||||||
|
value = strings.TrimSpace(value)
|
||||||
|
if value != "" {
|
||||||
|
result = append(result, value, value+"s")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if entry.Plural != nil {
|
||||||
|
if plural := strings.TrimSpace(*entry.Plural); plural != "" {
|
||||||
|
result = append(result, plural)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
30
internal/core/glossary/vocabulary_test.go
Normal file
30
internal/core/glossary/vocabulary_test.go
Normal file
@@ -0,0 +1,30 @@
|
|||||||
|
package glossary
|
||||||
|
|
||||||
|
import (
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestVocabularyKeepsContributorsAndExemptsOnlyActiveSpellings(t *testing.T) {
|
||||||
|
plural := "Aegides"
|
||||||
|
active := []Entry{{Collection: "spells", ID: "shield", Name: "Shield", Aliases: []string{"Ward"}}}
|
||||||
|
protected := []Entry{
|
||||||
|
active[0],
|
||||||
|
{Collection: "items", ID: "shield", Name: "shield", Aliases: []string{"Aegis"}, Plural: &plural},
|
||||||
|
}
|
||||||
|
full := Vocabulary(protected, nil)
|
||||||
|
if !reflect.DeepEqual(full[0].Identities, []Identity{{Collection: "spells", ID: "shield"}, {Collection: "items", ID: "shield"}}) {
|
||||||
|
t.Fatalf("shared spelling contributors: %+v", full[0])
|
||||||
|
}
|
||||||
|
reduced := Vocabulary(protected, active)
|
||||||
|
var got []string
|
||||||
|
for _, term := range reduced {
|
||||||
|
got = append(got, term.Spelling)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(got, []string{"Aegis", "Aegiss", "Aegides"}) {
|
||||||
|
t.Fatalf("remaining spellings: %v", got)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(full, Vocabulary(protected, nil)) || protected[0].Aliases[0] != "Ward" {
|
||||||
|
t.Fatal("deriving an exemption changed another view or its inputs")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
|
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||||
@@ -65,7 +66,8 @@ type ExecutionContext struct {
|
|||||||
ReportDiagnosticError func(error) `json:"-"`
|
ReportDiagnosticError func(error) `json:"-"`
|
||||||
Config *config.Config `json:"-"`
|
Config *config.Config `json:"-"`
|
||||||
WorkingTranscript *schema.Transcript `json:"-"`
|
WorkingTranscript *schema.Transcript `json:"-"`
|
||||||
Glossary *schema.Glossary `json:"-"`
|
Glossary []glossary.Entry `json:"-"`
|
||||||
|
ProtectedTerms []glossary.Term `json:"-"`
|
||||||
Section *SectionMetadata `json:"section,omitempty"`
|
Section *SectionMetadata `json:"section,omitempty"`
|
||||||
DiagnosticsDir string `json:"diagnostics_dir,omitempty"`
|
DiagnosticsDir string `json:"diagnostics_dir,omitempty"`
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -57,7 +57,6 @@ type Request struct {
|
|||||||
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
|
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
|
||||||
WorkingTranscript *schema.Transcript `json:"-"`
|
WorkingTranscript *schema.Transcript `json:"-"`
|
||||||
Section *contracts.SectionMetadata `json:"section,omitempty"`
|
Section *contracts.SectionMetadata `json:"section,omitempty"`
|
||||||
Glossary *schema.Glossary `json:"-"`
|
|
||||||
Config *config.Config `json:"-"`
|
Config *config.Config `json:"-"`
|
||||||
PromptID string `json:"prompt_id"`
|
PromptID string `json:"prompt_id"`
|
||||||
Variables map[string]string `json:"variables"`
|
Variables map[string]string `json:"variables"`
|
||||||
|
|||||||
@@ -0,0 +1,27 @@
|
|||||||
|
package proposal_generation
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestProposalContextPreservesDistinctEntries(t *testing.T) {
|
||||||
|
entries := []glossary.Entry{
|
||||||
|
{Collection: "spells", ID: "shield", Name: "Shield", Category: "spell", Description: "A spell"},
|
||||||
|
{Collection: "items", ID: "shield", Name: "Shield", Category: "item", Description: "An item"},
|
||||||
|
}
|
||||||
|
variables, err := BuildProposalVariables(&schema.Transcript{}, entries, 0, "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var decoded []glossary.Entry
|
||||||
|
if err := json.Unmarshal([]byte(variables["GlossaryJSON"]), &decoded); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(decoded) != 2 || decoded[0].Category != "spell" || decoded[1].Category != "item" {
|
||||||
|
t.Fatalf("prompt entries=%+v", decoded)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -5,13 +5,14 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/stagename"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/stagename"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
|
||||||
)
|
)
|
||||||
|
|
||||||
type ProposalVariableBuilder func(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) (map[string]string, error)
|
type ProposalVariableBuilder func(transcript *schema.Transcript, entries []glossary.Entry, sectionIndex int, transcriptDescription string) (map[string]string, error)
|
||||||
|
|
||||||
type ModuleProposalRequest struct {
|
type ModuleProposalRequest struct {
|
||||||
ProposalRequest contracts.ProposalRequest
|
ProposalRequest contracts.ProposalRequest
|
||||||
@@ -60,7 +61,6 @@ func ExecuteModuleProposal(ctx context.Context, req ModuleProposalRequest) (cont
|
|||||||
ReplacementPolicy: req.ProposalRequest.RunSpec.ReplacementPolicy,
|
ReplacementPolicy: req.ProposalRequest.RunSpec.ReplacementPolicy,
|
||||||
WorkingTranscript: req.ProposalRequest.WorkingTranscript,
|
WorkingTranscript: req.ProposalRequest.WorkingTranscript,
|
||||||
Section: req.ProposalRequest.Section,
|
Section: req.ProposalRequest.Section,
|
||||||
Glossary: req.ProposalRequest.Glossary,
|
|
||||||
Config: req.ProposalRequest.Config,
|
Config: req.ProposalRequest.Config,
|
||||||
PromptID: promptMetadata.PromptID,
|
PromptID: promptMetadata.PromptID,
|
||||||
Variables: variables,
|
Variables: variables,
|
||||||
|
|||||||
@@ -4,14 +4,15 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/promptcontext"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/promptcontext"
|
||||||
)
|
)
|
||||||
|
|
||||||
// BuildProposalVariables preserves the domain inputs while PromptKit renders
|
// BuildProposalVariables preserves the domain inputs while PromptKit renders
|
||||||
// the selected embedded proposal definition.
|
// the selected embedded proposal definition.
|
||||||
func BuildProposalVariables(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) (map[string]string, error) {
|
func BuildProposalVariables(transcript *schema.Transcript, entries []glossary.Entry, sectionIndex int, transcriptDescription string) (map[string]string, error) {
|
||||||
glossaryJSON, err := json.MarshalIndent(glossary, "", " ")
|
glossaryJSON, err := json.MarshalIndent(entries, "", " ")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("marshal glossary prompt context: %w", err)
|
return nil, fmt.Errorf("marshal glossary prompt context: %w", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -43,6 +43,7 @@ func ApplyProposals(
|
|||||||
transcript *schema.Transcript,
|
transcript *schema.Transcript,
|
||||||
proposals []EnrichedCorrectionProposal,
|
proposals []EnrichedCorrectionProposal,
|
||||||
policy ReplacementPolicy,
|
policy ReplacementPolicy,
|
||||||
|
allowsChange func(before, after string) bool,
|
||||||
) ApplyResult {
|
) ApplyResult {
|
||||||
working := cloneTranscript(transcript)
|
working := cloneTranscript(transcript)
|
||||||
ordered := sortProposalsByIndex(proposals)
|
ordered := sortProposalsByIndex(proposals)
|
||||||
@@ -68,6 +69,11 @@ func ApplyProposals(
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if allowsChange != nil && !allowsChange(working.Segments[segmentIdx].Text, preview.CorrectedSegmentText) {
|
||||||
|
skipped = append(skipped, newSkippedChange(proposal, SkipReasonProtectedTerm))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
working.Segments[segmentIdx].Text = preview.CorrectedSegmentText
|
working.Segments[segmentIdx].Text = preview.CorrectedSegmentText
|
||||||
applied = append(applied, AppliedChange{
|
applied = append(applied, AppliedChange{
|
||||||
ProposalIndex: proposal.ProposalIndex,
|
ProposalIndex: proposal.ProposalIndex,
|
||||||
@@ -149,6 +155,8 @@ func skipReasonMessage(reason ProposalSkipReason) string {
|
|||||||
return "original_text and corrected_text are identical"
|
return "original_text and corrected_text are identical"
|
||||||
case SkipReasonEmptyResultingText:
|
case SkipReasonEmptyResultingText:
|
||||||
return "proposal would leave the segment empty"
|
return "proposal would leave the segment empty"
|
||||||
|
case SkipReasonProtectedTerm:
|
||||||
|
return "proposal may alter protected glossary terminology"
|
||||||
case SkipReasonInvalidProposal:
|
case SkipReasonInvalidProposal:
|
||||||
return "proposal failed structural or policy validation"
|
return "proposal failed structural or policy validation"
|
||||||
default:
|
default:
|
||||||
|
|||||||
@@ -38,7 +38,7 @@ func TestApplyProposalsFixtureTranscriptGoldens(t *testing.T) {
|
|||||||
t.Fatalf("failed to parse fixture policy: %v", err)
|
t.Fatalf("failed to parse fixture policy: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
result := ApplyProposals(&fixture.Transcript, fixture.Proposals, policy)
|
result := ApplyProposals(&fixture.Transcript, fixture.Proposals, policy, nil)
|
||||||
assertJSONSemanticallyEqualFromGolden(t, result.Transcript, name+".transcript.golden.json")
|
assertJSONSemanticallyEqualFromGolden(t, result.Transcript, name+".transcript.golden.json")
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -62,7 +62,7 @@ func TestApplyProposalsFixtureSkipCoverage(t *testing.T) {
|
|||||||
t.Fatalf("failed to parse fixture policy: %v", err)
|
t.Fatalf("failed to parse fixture policy: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
result := ApplyProposals(&fixture.Transcript, fixture.Proposals, policy)
|
result := ApplyProposals(&fixture.Transcript, fixture.Proposals, policy, nil)
|
||||||
if len(result.Skipped) == 0 {
|
if len(result.Skipped) == 0 {
|
||||||
t.Fatal("expected at least one skipped change")
|
t.Fatal("expected at least one skipped change")
|
||||||
}
|
}
|
||||||
@@ -81,7 +81,7 @@ func TestApplyProposalsFixtureMixedScenarioRecordGoldens(t *testing.T) {
|
|||||||
t.Fatalf("failed to parse fixture policy: %v", err)
|
t.Fatalf("failed to parse fixture policy: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
result := ApplyProposals(&fixture.Transcript, fixture.Proposals, policy)
|
result := ApplyProposals(&fixture.Transcript, fixture.Proposals, policy, nil)
|
||||||
payload := recordsPayload{
|
payload := recordsPayload{
|
||||||
AppliedChanges: result.Applied,
|
AppliedChanges: result.Applied,
|
||||||
SkippedChanges: result.Skipped,
|
SkippedChanges: result.Skipped,
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ func TestApplyProposalsApplyingOneProposal(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
result := ApplyProposals(transcript, proposals, ReplacementPolicyRequireUnique)
|
result := ApplyProposals(transcript, proposals, ReplacementPolicyRequireUnique, nil)
|
||||||
|
|
||||||
if got := result.Transcript.Segments[0].Text; got != "Jesters" {
|
if got := result.Transcript.Segments[0].Text; got != "Jesters" {
|
||||||
t.Fatalf("expected corrected text Jesters, got %q", got)
|
t.Fatalf("expected corrected text Jesters, got %q", got)
|
||||||
@@ -57,7 +57,7 @@ func TestApplyProposalsApplyingMultipleProposalsInStableOrder(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
result := ApplyProposals(transcript, proposals, ReplacementPolicyReplaceAll)
|
result := ApplyProposals(transcript, proposals, ReplacementPolicyReplaceAll, nil)
|
||||||
|
|
||||||
if got := result.Transcript.Segments[0].Text; got != "X X" {
|
if got := result.Transcript.Segments[0].Text; got != "X X" {
|
||||||
t.Fatalf("expected final text X X, got %q", got)
|
t.Fatalf("expected final text X X, got %q", got)
|
||||||
@@ -93,7 +93,7 @@ func TestApplyProposalsStaleProposalSkippedAfterEarlierChange(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
result := ApplyProposals(transcript, proposals, ReplacementPolicyRequireUnique)
|
result := ApplyProposals(transcript, proposals, ReplacementPolicyRequireUnique, nil)
|
||||||
|
|
||||||
if got := result.Transcript.Segments[0].Text; got != "foo-bar" {
|
if got := result.Transcript.Segments[0].Text; got != "foo-bar" {
|
||||||
t.Fatalf("expected final text foo-bar, got %q", got)
|
t.Fatalf("expected final text foo-bar, got %q", got)
|
||||||
@@ -120,7 +120,7 @@ func TestApplyProposalsRechecksBlankResultAfterEarlierDeletion(t *testing.T) {
|
|||||||
t.Fatalf("proposal must be initially applicable: %+v", preview)
|
t.Fatalf("proposal must be initially applicable: %+v", preview)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
result := ApplyProposals(transcript, proposed, ReplacementPolicyRequireUnique)
|
result := ApplyProposals(transcript, proposed, ReplacementPolicyRequireUnique, nil)
|
||||||
if got := result.Transcript.Segments[0].Text; got != " b" {
|
if got := result.Transcript.Segments[0].Text; got != " b" {
|
||||||
t.Fatalf("expected literal surviving text, got %q", got)
|
t.Fatalf("expected literal surviving text, got %q", got)
|
||||||
}
|
}
|
||||||
@@ -143,7 +143,7 @@ func TestApplyProposalsMissingSegmentSkip(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
result := ApplyProposals(transcript, proposals, ReplacementPolicyRequireUnique)
|
result := ApplyProposals(transcript, proposals, ReplacementPolicyRequireUnique, nil)
|
||||||
|
|
||||||
if len(result.Skipped) != 1 {
|
if len(result.Skipped) != 1 {
|
||||||
t.Fatalf("expected 1 skipped change, got %d", len(result.Skipped))
|
t.Fatalf("expected 1 skipped change, got %d", len(result.Skipped))
|
||||||
@@ -167,7 +167,7 @@ func TestApplyProposalsRequireUniqueAmbiguousSkip(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
result := ApplyProposals(transcript, proposals, ReplacementPolicyRequireUnique)
|
result := ApplyProposals(transcript, proposals, ReplacementPolicyRequireUnique, nil)
|
||||||
|
|
||||||
if len(result.Skipped) != 1 {
|
if len(result.Skipped) != 1 {
|
||||||
t.Fatalf("expected 1 skipped change, got %d", len(result.Skipped))
|
t.Fatalf("expected 1 skipped change, got %d", len(result.Skipped))
|
||||||
@@ -191,7 +191,7 @@ func TestApplyProposalsReplaceAllMultipleReplacements(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
result := ApplyProposals(transcript, proposals, ReplacementPolicyReplaceAll)
|
result := ApplyProposals(transcript, proposals, ReplacementPolicyReplaceAll, nil)
|
||||||
|
|
||||||
if len(result.Applied) != 1 {
|
if len(result.Applied) != 1 {
|
||||||
t.Fatalf("expected 1 applied change, got %d", len(result.Applied))
|
t.Fatalf("expected 1 applied change, got %d", len(result.Applied))
|
||||||
@@ -220,7 +220,7 @@ func TestApplyProposalsNoMutationOfInputTranscript(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
_ = ApplyProposals(transcript, proposals, ReplacementPolicyRequireUnique)
|
_ = ApplyProposals(transcript, proposals, ReplacementPolicyRequireUnique, nil)
|
||||||
|
|
||||||
if !reflect.DeepEqual(transcript, before) {
|
if !reflect.DeepEqual(transcript, before) {
|
||||||
t.Fatalf("input transcript mutated: before=%+v after=%+v", before, transcript)
|
t.Fatalf("input transcript mutated: before=%+v after=%+v", before, transcript)
|
||||||
@@ -249,7 +249,7 @@ func TestApplyProposalsSegmentMetadataPreservation(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
result := ApplyProposals(transcript, proposals, ReplacementPolicyRequireUnique)
|
result := ApplyProposals(transcript, proposals, ReplacementPolicyRequireUnique, nil)
|
||||||
seg := result.Transcript.Segments[0]
|
seg := result.Transcript.Segments[0]
|
||||||
|
|
||||||
if seg.Speaker != "A" || seg.Start != 12.5 || seg.End != 15.0 {
|
if seg.Speaker != "A" || seg.Start != 12.5 || seg.End != 15.0 {
|
||||||
@@ -283,7 +283,7 @@ func TestApplyProposalsAppliedAndSkippedRecords(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
result := ApplyProposals(transcript, proposals, ReplacementPolicyRequireUnique)
|
result := ApplyProposals(transcript, proposals, ReplacementPolicyRequireUnique, nil)
|
||||||
|
|
||||||
if len(result.Applied) != 1 || len(result.Skipped) != 1 {
|
if len(result.Applied) != 1 || len(result.Skipped) != 1 {
|
||||||
t.Fatalf("expected 1 applied and 1 skipped, got applied=%d skipped=%d", len(result.Applied), len(result.Skipped))
|
t.Fatalf("expected 1 applied and 1 skipped, got applied=%d skipped=%d", len(result.Applied), len(result.Skipped))
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ const (
|
|||||||
SkipReasonNoEffect ProposalSkipReason = "no_effect"
|
SkipReasonNoEffect ProposalSkipReason = "no_effect"
|
||||||
SkipReasonEmptyResultingText ProposalSkipReason = "empty_resulting_segment"
|
SkipReasonEmptyResultingText ProposalSkipReason = "empty_resulting_segment"
|
||||||
SkipReasonInvalidProposal ProposalSkipReason = "invalid_proposal"
|
SkipReasonInvalidProposal ProposalSkipReason = "invalid_proposal"
|
||||||
|
SkipReasonProtectedTerm ProposalSkipReason = "protected_glossary_term"
|
||||||
)
|
)
|
||||||
|
|
||||||
// SegmentPreviewResult is a deterministic preview of applying one proposal to
|
// SegmentPreviewResult is a deterministic preview of applying one proposal to
|
||||||
|
|||||||
@@ -136,17 +136,12 @@ func (v NoEffectValidator) Validate(_ context.Context, req Request) (Result, err
|
|||||||
return Result{ValidatorName: v.Name(), Decisions: decisions}, nil
|
return Result{ValidatorName: v.Name(), Decisions: decisions}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func ValidateProtectedTerms(req Request, glossaryStage bool) (Result, error) {
|
func ValidateProtectedTerms(req Request) (Result, error) {
|
||||||
vocab := NewProtectedVocabulary(req.Glossary)
|
vocab := NewProtectedVocabulary(req.ProtectedTerms)
|
||||||
decisions := make([]Decision, 0, len(req.CandidateProposal))
|
decisions := make([]Decision, 0, len(req.CandidateProposal))
|
||||||
for _, c := range req.CandidateProposal {
|
for _, c := range req.CandidateProposal {
|
||||||
var reason string
|
reason := vocab.violationReason(c.OriginalText, c.CorrectedText)
|
||||||
if glossaryStage {
|
if reason != "" && req.ModuleKey == modulecatalog.KeySpokenWord && req.WorkingTranscript != nil {
|
||||||
reason = vocab.glossaryStageViolationReason(c.OriginalText, c.CorrectedText)
|
|
||||||
} else {
|
|
||||||
reason = vocab.violationReason(c.OriginalText, c.CorrectedText)
|
|
||||||
}
|
|
||||||
if reason != "" && !glossaryStage && req.ModuleKey == modulecatalog.KeySpokenWord && req.WorkingTranscript != nil {
|
|
||||||
for i := range req.WorkingTranscript.Segments {
|
for i := range req.WorkingTranscript.Segments {
|
||||||
segment := &req.WorkingTranscript.Segments[i]
|
segment := &req.WorkingTranscript.Segments[i]
|
||||||
if segment.ID != c.TargetSegmentID {
|
if segment.ID != c.TargetSegmentID {
|
||||||
|
|||||||
@@ -2,13 +2,14 @@ package validators
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||||
)
|
)
|
||||||
|
|
||||||
// validationContext supplies adjacent source segments within the current chunk
|
// validationContext supplies adjacent source segments within the current chunk
|
||||||
// and glossary definitions mentioned in the candidate or that local context.
|
// and glossary definitions mentioned in the candidate or that local context.
|
||||||
// These fields travel in each item so batching accounts for their size.
|
// These fields travel in each item so batching accounts for their size.
|
||||||
func validationContext(segments []schema.Segment, position int, glossary *schema.Glossary, vocabulary ProtectedVocabulary, corrected string) ([]NeighboringSegment, []schema.GlossaryEntry) {
|
func validationContext(segments []schema.Segment, position int, selected []glossary.Entry, vocabulary ProtectedVocabulary, corrected string) ([]NeighboringSegment, []glossary.Entry) {
|
||||||
var neighbors []NeighboringSegment
|
var neighbors []NeighboringSegment
|
||||||
source := segments[position].Text
|
source := segments[position].Text
|
||||||
if position > 0 {
|
if position > 0 {
|
||||||
@@ -22,16 +23,16 @@ func validationContext(segments []schema.Segment, position int, glossary *schema
|
|||||||
for _, neighbor := range neighbors {
|
for _, neighbor := range neighbors {
|
||||||
texts = append(texts, neighbor.Text)
|
texts = append(texts, neighbor.Text)
|
||||||
}
|
}
|
||||||
var entries []schema.GlossaryEntry
|
var entries []glossary.Entry
|
||||||
if glossary != nil {
|
if selected != nil {
|
||||||
mentioned := make(map[int]bool)
|
mentioned := make(map[glossary.Identity]bool)
|
||||||
for _, text := range texts {
|
for _, text := range texts {
|
||||||
for identity := range vocabulary.occurrencesByIdentity(text) {
|
for identity := range vocabulary.occurrencesByIdentity(text) {
|
||||||
mentioned[identity] = true
|
mentioned[identity] = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for i, entry := range glossary.Entries {
|
for _, entry := range selected {
|
||||||
if mentioned[i] {
|
if mentioned[glossary.Identity{Collection: entry.Collection, ID: entry.ID}] {
|
||||||
entries = append(entries, entry)
|
entries = append(entries, entry)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
package validators
|
package validators
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -20,15 +20,15 @@ type StructuredLLMClient = structuredoutput.Client
|
|||||||
// LLMValidationItem retains the internal proposal index until prompt encoding.
|
// LLMValidationItem retains the internal proposal index until prompt encoding.
|
||||||
// BuildValidationVariables substitutes a one-based reference for each batch.
|
// BuildValidationVariables substitutes a one-based reference for each batch.
|
||||||
type LLMValidationItem struct {
|
type LLMValidationItem struct {
|
||||||
CorrectionIndex int `json:"correction_index"`
|
CorrectionIndex int `json:"correction_index"`
|
||||||
SegmentID int `json:"-"`
|
SegmentID int `json:"-"`
|
||||||
OriginalText string `json:"original_text"`
|
OriginalText string `json:"original_text"`
|
||||||
CorrectedText string `json:"corrected_text"`
|
CorrectedText string `json:"corrected_text"`
|
||||||
OriginalSegmentText string `json:"original_segment_text"`
|
OriginalSegmentText string `json:"original_segment_text"`
|
||||||
CorrectedSegmentText string `json:"corrected_segment_text"`
|
CorrectedSegmentText string `json:"corrected_segment_text"`
|
||||||
Categories []string `json:"categories,omitempty"`
|
Categories []string `json:"categories,omitempty"`
|
||||||
NeighboringSegments []NeighboringSegment `json:"neighboring_segments,omitempty"`
|
NeighboringSegments []NeighboringSegment `json:"neighboring_segments,omitempty"`
|
||||||
GlossaryEntries []schema.GlossaryEntry `json:"glossary_entries,omitempty"`
|
GlossaryEntries []glossary.Entry `json:"glossary_entries,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// LLMValidationRequest is the canonical request model for LLM-backed validators.
|
// LLMValidationRequest is the canonical request model for LLM-backed validators.
|
||||||
@@ -38,7 +38,6 @@ type LLMValidationRequest struct {
|
|||||||
ModuleKey string `json:"module_key"`
|
ModuleKey string `json:"module_key"`
|
||||||
ModuleInstance string `json:"module_instance"`
|
ModuleInstance string `json:"module_instance"`
|
||||||
ReplacementPolicy string `json:"replacement_policy"`
|
ReplacementPolicy string `json:"replacement_policy"`
|
||||||
Glossary *schema.Glossary `json:"-"`
|
|
||||||
Items []LLMValidationItem `json:"corrections"`
|
Items []LLMValidationItem `json:"corrections"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import (
|
|||||||
|
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||||
@@ -253,7 +254,7 @@ func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorT
|
|||||||
immediate := make([]Decision, 0)
|
immediate := make([]Decision, 0)
|
||||||
var segments []schema.Segment
|
var segments []schema.Segment
|
||||||
positions := make(map[int]int)
|
positions := make(map[int]int)
|
||||||
vocabulary := NewProtectedVocabulary(req.Glossary)
|
vocabulary := NewProtectedVocabulary(glossary.Vocabulary(req.Glossary, nil))
|
||||||
if req.WorkingTranscript != nil {
|
if req.WorkingTranscript != nil {
|
||||||
segments = req.WorkingTranscript.Segments
|
segments = req.WorkingTranscript.Segments
|
||||||
for i, seg := range segments {
|
for i, seg := range segments {
|
||||||
@@ -295,7 +296,6 @@ func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorT
|
|||||||
ModuleKey: req.ModuleKey,
|
ModuleKey: req.ModuleKey,
|
||||||
ModuleInstance: req.ModuleInstance,
|
ModuleInstance: req.ModuleInstance,
|
||||||
ReplacementPolicy: string(req.ReplacementPolicy),
|
ReplacementPolicy: string(req.ReplacementPolicy),
|
||||||
Glossary: req.Glossary,
|
|
||||||
Items: items,
|
Items: items,
|
||||||
}, immediate
|
}, immediate
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||||
@@ -41,7 +42,8 @@ type Request struct {
|
|||||||
ModuleInstance string `json:"module_instance"`
|
ModuleInstance string `json:"module_instance"`
|
||||||
ValidatorOrder *int `json:"validator_order,omitempty"`
|
ValidatorOrder *int `json:"validator_order,omitempty"`
|
||||||
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
|
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
|
||||||
Glossary *schema.Glossary `json:"-"`
|
Glossary []glossary.Entry `json:"-"`
|
||||||
|
ProtectedTerms []glossary.Term `json:"-"`
|
||||||
Config *config.Config `json:"-"`
|
Config *config.Config `json:"-"`
|
||||||
LLMClient StructuredLLMClient `json:"-"`
|
LLMClient StructuredLLMClient `json:"-"`
|
||||||
DiagnosticsWriter InteractionDiagnosticsWriter `json:"-"`
|
DiagnosticsWriter InteractionDiagnosticsWriter `json:"-"`
|
||||||
|
|||||||
@@ -5,17 +5,17 @@ import (
|
|||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
|
||||||
)
|
)
|
||||||
|
|
||||||
type protectedTermDef struct {
|
type protectedTermDef struct {
|
||||||
identity int
|
identities []glossary.Identity
|
||||||
canonical string
|
canonical string
|
||||||
}
|
}
|
||||||
|
|
||||||
type protectedOccurrence struct {
|
type protectedOccurrence struct {
|
||||||
text string
|
text string
|
||||||
identity int
|
identity glossary.Identity
|
||||||
canonical string
|
canonical string
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -25,47 +25,13 @@ type ProtectedVocabulary struct {
|
|||||||
pattern *regexp.Regexp
|
pattern *regexp.Regexp
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExtractProtectedTerms returns a stable, de-duplicated list of protected terms
|
// NewProtectedVocabulary prepares the independently selected terms for matching.
|
||||||
// derived from glossary names, aliases, synthetic plural forms, and explicit
|
func NewProtectedVocabulary(terms []glossary.Term) ProtectedVocabulary {
|
||||||
// plural fields.
|
termsByFolded := make(map[string]protectedTermDef, len(terms))
|
||||||
func ExtractProtectedTerms(glossary *schema.Glossary) []string {
|
alternatives := make([]string, 0, len(terms))
|
||||||
vocab := NewProtectedVocabulary(glossary)
|
for _, term := range terms {
|
||||||
out := make([]string, 0, len(vocab.termsByFolded))
|
termsByFolded[strings.ToLower(term.Spelling)] = protectedTermDef{identities: term.Identities, canonical: term.Spelling}
|
||||||
for _, def := range vocab.termsByFolded {
|
alternatives = append(alternatives, regexp.QuoteMeta(term.Spelling))
|
||||||
out = append(out, def.canonical)
|
|
||||||
}
|
|
||||||
sort.SliceStable(out, func(i, j int) bool {
|
|
||||||
li := strings.ToLower(out[i])
|
|
||||||
lj := strings.ToLower(out[j])
|
|
||||||
if li == lj {
|
|
||||||
return out[i] < out[j]
|
|
||||||
}
|
|
||||||
return li < lj
|
|
||||||
})
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewProtectedVocabulary builds a deterministic protected vocabulary from a glossary.
|
|
||||||
func NewProtectedVocabulary(glossary *schema.Glossary) ProtectedVocabulary {
|
|
||||||
termsByFolded := make(map[string]protectedTermDef)
|
|
||||||
if glossary != nil {
|
|
||||||
for identity, entry := range glossary.Entries {
|
|
||||||
entryTerms := append([]string{entry.Name}, entry.Aliases...)
|
|
||||||
for _, term := range entryTerms {
|
|
||||||
trimmed := strings.TrimSpace(term)
|
|
||||||
if trimmed == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
addProtectedTerm(termsByFolded, trimmed, identity)
|
|
||||||
addProtectedTerm(termsByFolded, trimmed+"s", identity)
|
|
||||||
}
|
|
||||||
addProtectedTerm(termsByFolded, entry.Plural, identity)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
alternatives := make([]string, 0, len(termsByFolded))
|
|
||||||
for _, def := range termsByFolded {
|
|
||||||
alternatives = append(alternatives, regexp.QuoteMeta(def.canonical))
|
|
||||||
}
|
}
|
||||||
sort.SliceStable(alternatives, func(i, j int) bool { return len(alternatives[i]) > len(alternatives[j]) })
|
sort.SliceStable(alternatives, func(i, j int) bool { return len(alternatives[i]) > len(alternatives[j]) })
|
||||||
if len(alternatives) == 0 {
|
if len(alternatives) == 0 {
|
||||||
@@ -75,16 +41,13 @@ func NewProtectedVocabulary(glossary *schema.Glossary) ProtectedVocabulary {
|
|||||||
return ProtectedVocabulary{termsByFolded: termsByFolded, pattern: pattern}
|
return ProtectedVocabulary{termsByFolded: termsByFolded, pattern: pattern}
|
||||||
}
|
}
|
||||||
|
|
||||||
func addProtectedTerm(terms map[string]protectedTermDef, term string, identity int) {
|
// AllowsChange checks a proposed segment against the effective protection view.
|
||||||
trimmed := strings.TrimSpace(term)
|
// Spoken-word cleanup may reduce repeated mentions while preserving each identity.
|
||||||
if trimmed == "" {
|
func (v ProtectedVocabulary) AllowsChange(before, after string, allowRepetitionCleanup bool) bool {
|
||||||
return
|
if v.violationReason(before, after) == "" {
|
||||||
|
return true
|
||||||
}
|
}
|
||||||
folded := strings.ToLower(trimmed)
|
return allowRepetitionCleanup && v.preservesRepeatedIdentities(before, after)
|
||||||
if _, ok := terms[folded]; ok {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
terms[folded] = protectedTermDef{identity: identity, canonical: trimmed}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (v ProtectedVocabulary) violationReason(before, after string) string {
|
func (v ProtectedVocabulary) violationReason(before, after string) string {
|
||||||
@@ -99,20 +62,8 @@ func (v ProtectedVocabulary) violationReason(before, after string) string {
|
|||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
func (v ProtectedVocabulary) glossaryStageViolationReason(before, after string) string {
|
func (v ProtectedVocabulary) occurrencesByIdentity(text string) map[glossary.Identity][]protectedOccurrence {
|
||||||
beforeByID := v.occurrencesByIdentity(before)
|
byID := make(map[glossary.Identity][]protectedOccurrence)
|
||||||
afterByID := v.occurrencesByIdentity(after)
|
|
||||||
if reason := validateGlossaryStageIdentityPreservation(beforeByID, afterByID); reason != "" {
|
|
||||||
return reason
|
|
||||||
}
|
|
||||||
if reason := validateCapitalizationTransitions(beforeByID, afterByID); reason != "" {
|
|
||||||
return reason
|
|
||||||
}
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
func (v ProtectedVocabulary) occurrencesByIdentity(text string) map[int][]protectedOccurrence {
|
|
||||||
byID := make(map[int][]protectedOccurrence)
|
|
||||||
for _, o := range v.occurrences(text) {
|
for _, o := range v.occurrences(text) {
|
||||||
byID[o.identity] = append(byID[o.identity], o)
|
byID[o.identity] = append(byID[o.identity], o)
|
||||||
}
|
}
|
||||||
@@ -134,16 +85,14 @@ func (v ProtectedVocabulary) occurrences(text string) []protectedOccurrence {
|
|||||||
if !ok {
|
if !ok {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
out = append(out, protectedOccurrence{
|
for _, identity := range def.identities {
|
||||||
text: matched,
|
out = append(out, protectedOccurrence{text: matched, identity: identity, canonical: def.canonical})
|
||||||
identity: def.identity,
|
}
|
||||||
canonical: def.canonical,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateIdentityPreservation(beforeByID, afterByID map[int][]protectedOccurrence) string {
|
func validateIdentityPreservation(beforeByID, afterByID map[glossary.Identity][]protectedOccurrence) string {
|
||||||
for identity, beforeItems := range beforeByID {
|
for identity, beforeItems := range beforeByID {
|
||||||
if len(afterByID[identity]) < len(beforeItems) {
|
if len(afterByID[identity]) < len(beforeItems) {
|
||||||
return "proposal may alter protected glossary terminology"
|
return "proposal may alter protected glossary terminology"
|
||||||
@@ -152,22 +101,7 @@ func validateIdentityPreservation(beforeByID, afterByID map[int][]protectedOccur
|
|||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateGlossaryStageIdentityPreservation(beforeByID, afterByID map[int][]protectedOccurrence) string {
|
func validateCapitalizationTransitions(beforeByID, afterByID map[glossary.Identity][]protectedOccurrence) string {
|
||||||
beforeTotal := 0
|
|
||||||
for _, items := range beforeByID {
|
|
||||||
beforeTotal += len(items)
|
|
||||||
}
|
|
||||||
afterTotal := 0
|
|
||||||
for _, items := range afterByID {
|
|
||||||
afterTotal += len(items)
|
|
||||||
}
|
|
||||||
if afterTotal < beforeTotal {
|
|
||||||
return "proposal may alter protected glossary terminology"
|
|
||||||
}
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
func validateCapitalizationTransitions(beforeByID, afterByID map[int][]protectedOccurrence) string {
|
|
||||||
for identity, afterItems := range afterByID {
|
for identity, afterItems := range afterByID {
|
||||||
beforeItems := beforeByID[identity]
|
beforeItems := beforeByID[identity]
|
||||||
for i, afterItem := range afterItems {
|
for i, afterItem := range afterItems {
|
||||||
|
|||||||
75
internal/framework/validators/vocabulary_test.go
Normal file
75
internal/framework/validators/vocabulary_test.go
Normal file
@@ -0,0 +1,75 @@
|
|||||||
|
package validators
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||||
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestIndependentProtectionForSelectedGlossary(t *testing.T) {
|
||||||
|
active := []glossary.Entry{{Collection: "spells", ID: "shield", Name: "Shield"}}
|
||||||
|
entries := append(append([]glossary.Entry(nil), active...), glossary.Entry{Collection: "items", ID: "shield", Name: "Shield", Aliases: []string{"Aegis"}})
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
terms []glossary.Term
|
||||||
|
before, after string
|
||||||
|
approved bool
|
||||||
|
}{
|
||||||
|
{"shared spelling exempt", glossary.Vocabulary(entries, active), "Shield", "Barrier", true},
|
||||||
|
{"other alias protected", glossary.Vocabulary(entries, active), "Aegis", "Armor", false},
|
||||||
|
{"full protection retained", glossary.Vocabulary(entries, nil), "Shield", "Barrier", false},
|
||||||
|
{"empty protection", nil, "Aegis", "Armor", true},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
result, err := ValidateProtectedTerms(Request{ProtectedTerms: tc.terms, CandidateProposal: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{OriginalText: tc.before, CorrectedText: tc.after}}}})
|
||||||
|
if err != nil || len(result.Decisions) != 1 || result.Decisions[0].Approved != tc.approved {
|
||||||
|
t.Fatalf("result=%+v err=%v", result, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSharedAliasRetainsAllIdentities(t *testing.T) {
|
||||||
|
entries := []glossary.Entry{
|
||||||
|
{Collection: "a", ID: "first", Name: "First", Aliases: []string{"Shared"}},
|
||||||
|
{Collection: "b", ID: "second", Name: "Second", Aliases: []string{"Shared"}},
|
||||||
|
}
|
||||||
|
vocabulary := NewProtectedVocabulary(glossary.Vocabulary(entries, nil))
|
||||||
|
identities := vocabulary.occurrencesByIdentity("Shared")
|
||||||
|
if len(identities) != 2 || len(identities[glossary.Identity{Collection: "a", ID: "first"}]) != 1 || len(identities[glossary.Identity{Collection: "b", ID: "second"}]) != 1 {
|
||||||
|
t.Fatalf("identities=%v", identities)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplicationChecksCurrentTextWithSelectedProtection(t *testing.T) {
|
||||||
|
active := []glossary.Entry{{Collection: "spells", ID: "spell", Name: "Shield"}}
|
||||||
|
protected := append(append([]glossary.Entry(nil), active...), glossary.Entry{Collection: "items", ID: "item", Name: "Shield", Aliases: []string{"Aegis"}})
|
||||||
|
vocabulary := NewProtectedVocabulary(glossary.Vocabulary(protected, active))
|
||||||
|
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Shield and Aegis"}}}
|
||||||
|
candidates := []proposals.EnrichedCorrectionProposal{
|
||||||
|
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "Shield", CorrectedText: "Barrier", Confidence: 1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}},
|
||||||
|
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "egis", CorrectedText: "rmor", Confidence: 1}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}},
|
||||||
|
}
|
||||||
|
result := proposals.ApplyProposals(transcript, candidates, proposals.ReplacementPolicyRequireUnique, func(before, after string) bool {
|
||||||
|
return vocabulary.AllowsChange(before, after, false)
|
||||||
|
})
|
||||||
|
if result.Transcript.Segments[0].Text != "Barrier and Aegis" || len(result.Applied) != 1 || len(result.Skipped) != 1 || result.Skipped[0].SkipReason != proposals.SkipReasonProtectedTerm {
|
||||||
|
t.Fatalf("unexpected application: %+v", result)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidationContextIncludesEachSharedDefinition(t *testing.T) {
|
||||||
|
entries := []glossary.Entry{
|
||||||
|
{Collection: "spells", ID: "shield", Name: "Shield", Category: "spell"},
|
||||||
|
{Collection: "items", ID: "shield", Name: "Shield", Category: "item"},
|
||||||
|
{Collection: "items", ID: "sword", Name: "Sword", Category: "item"},
|
||||||
|
}
|
||||||
|
segments := []schema.Segment{{ID: 1, Text: "Shield"}}
|
||||||
|
_, selected := validationContext(segments, 0, entries, NewProtectedVocabulary(glossary.Vocabulary(entries, nil)), "Barrier")
|
||||||
|
if len(selected) != 2 || selected[0].Category != "spell" || selected[1].ID != "shield" || selected[1].Collection != "items" {
|
||||||
|
t.Fatalf("selected=%+v", selected)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -6,7 +6,6 @@ import (
|
|||||||
|
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||||
"gitea.maximumdirect.net/eric/audita/v3/internal/validators/protected_terms"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var builtInChains = map[string][]string{
|
var builtInChains = map[string][]string{
|
||||||
@@ -75,16 +74,6 @@ func ResolveBuiltInChain(moduleKey string, registry *Registry) ([]contracts.Vali
|
|||||||
|
|
||||||
out := make([]contracts.Validator, 0, len(keys))
|
out := make([]contracts.Validator, 0, len(keys))
|
||||||
for _, key := range keys {
|
for _, key := range keys {
|
||||||
if moduleKey == modulecatalog.KeyGlossary && key == KeyProtectedTerms {
|
|
||||||
// Glossary stages preserve current stricter protection semantics while
|
|
||||||
// reporting the stable protected_terms key.
|
|
||||||
v, buildErr := protected_terms.NewGlossaryStage()
|
|
||||||
if buildErr != nil {
|
|
||||||
return nil, buildErr
|
|
||||||
}
|
|
||||||
out = append(out, v)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
v, buildErr := registry.MustBuild(key)
|
v, buildErr := registry.MustBuild(key)
|
||||||
if buildErr != nil {
|
if buildErr != nil {
|
||||||
return nil, buildErr
|
return nil, buildErr
|
||||||
|
|||||||
@@ -14,31 +14,10 @@ func New() (contracts.Validator, error) {
|
|||||||
return validatormetadata.Wrap(validator{}, validatormetadata.ExecutionClassDeterministic), nil
|
return validatormetadata.Wrap(validator{}, validatormetadata.ExecutionClassDeterministic), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewGlossaryStage() (contracts.Validator, error) {
|
type validator struct{}
|
||||||
return validatormetadata.Wrap(validator{glossaryStage: true}, validatormetadata.ExecutionClassDeterministic), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type validator struct {
|
func (v validator) Name() string { return Key }
|
||||||
glossaryStage bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func (v validator) Name() string {
|
|
||||||
return Key
|
|
||||||
}
|
|
||||||
|
|
||||||
func (v validator) Validate(_ context.Context, req contracts.ValidationRequest) (frameworkvalidators.Result, error) {
|
func (v validator) Validate(_ context.Context, req contracts.ValidationRequest) (frameworkvalidators.Result, error) {
|
||||||
if !v.glossaryStage && req.ModuleKey == "glossary" {
|
return frameworkvalidators.ValidateProtectedTerms(req)
|
||||||
decisions := make([]frameworkvalidators.Decision, 0, len(req.CandidateProposal))
|
|
||||||
for _, c := range req.CandidateProposal {
|
|
||||||
decisions = append(decisions, frameworkvalidators.Decision{
|
|
||||||
ProposalIndex: c.ProposalIndex,
|
|
||||||
Approved: true,
|
|
||||||
ReasonCode: frameworkvalidators.ReasonApproved,
|
|
||||||
Message: "approved",
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return frameworkvalidators.Result{ValidatorName: Key, Decisions: decisions}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return frameworkvalidators.ValidateProtectedTerms(req, v.glossaryStage)
|
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user