93 lines
3.3 KiB
Go
93 lines
3.3 KiB
Go
package validators
|
|
|
|
import (
|
|
"slices"
|
|
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
|
)
|
|
|
|
// GlossaryView holds the read-only matching data shared by a stage's validators
|
|
// and apply-time checks.
|
|
type GlossaryView struct {
|
|
Protection ProtectedVocabulary
|
|
context glossaryContext
|
|
}
|
|
|
|
func PrepareGlossaryView(entries []glossary.Entry, protected []glossary.Term) *GlossaryView {
|
|
return &GlossaryView{Protection: NewProtectedVocabulary(protected), context: prepareGlossaryContext(entries)}
|
|
}
|
|
|
|
type glossaryContext struct {
|
|
entries []glossary.Entry
|
|
positions map[glossary.Identity][]int
|
|
vocabulary ProtectedVocabulary
|
|
}
|
|
|
|
func prepareGlossaryContext(entries []glossary.Entry) glossaryContext {
|
|
positions := make(map[glossary.Identity][]int, len(entries))
|
|
for i, entry := range entries {
|
|
identity := glossary.Identity{Collection: entry.Collection, ID: entry.ID}
|
|
positions[identity] = append(positions[identity], i)
|
|
}
|
|
return glossaryContext{entries: entries, positions: positions, vocabulary: NewProtectedVocabulary(glossary.Vocabulary(entries, nil))}
|
|
}
|
|
|
|
// validationContext supplies adjacent source segments within the current chunk
|
|
// and glossary definitions mentioned in the candidate or that local context.
|
|
// These fields travel in each item so batching accounts for their size.
|
|
func (view glossaryContext) validationContext(segments []schema.Segment, position int, corrected string) ([]NeighboringSegment, []glossary.Entry) {
|
|
var neighbors []NeighboringSegment
|
|
source := segments[position].Text
|
|
if position > 0 {
|
|
neighbors = append(neighbors, neighboringSegment("preceding", segments[position-1]))
|
|
}
|
|
if position+1 < len(segments) {
|
|
neighbors = append(neighbors, neighboringSegment("following", segments[position+1]))
|
|
}
|
|
|
|
texts := []string{source, corrected}
|
|
for _, neighbor := range neighbors {
|
|
texts = append(texts, neighbor.Text)
|
|
}
|
|
mentioned := make(map[glossary.Identity]struct{})
|
|
for _, text := range texts {
|
|
for _, occurrence := range view.vocabulary.occurrences(text) {
|
|
mentioned[occurrence.identity] = struct{}{}
|
|
}
|
|
}
|
|
var matched []int
|
|
for identity := range mentioned {
|
|
matched = append(matched, view.positions[identity]...)
|
|
}
|
|
slices.Sort(matched)
|
|
var entries []glossary.Entry
|
|
for _, index := range matched {
|
|
entries = append(entries, view.entries[index])
|
|
}
|
|
return neighbors, entries
|
|
}
|
|
|
|
// fitValidationContext drops optional context before treating required candidate
|
|
// text as oversized. Glossary definitions take priority over adjacent segments.
|
|
func fitValidationContext(item LLMValidationItem, budget int, estimator chunking.TokenEstimator) LLMValidationItem {
|
|
for {
|
|
tokens, err := estimateBatchTokens(estimator, []LLMValidationItem{item})
|
|
if err != nil || tokens <= budget {
|
|
return item
|
|
}
|
|
if len(item.NeighboringSegments) > 0 {
|
|
item.NeighboringSegments = item.NeighboringSegments[:len(item.NeighboringSegments)-1]
|
|
} else if len(item.GlossaryEntries) > 0 {
|
|
item.GlossaryEntries = item.GlossaryEntries[:len(item.GlossaryEntries)-1]
|
|
} else {
|
|
return item
|
|
}
|
|
}
|
|
}
|
|
|
|
func neighboringSegment(position string, segment schema.Segment) NeighboringSegment {
|
|
return NeighboringSegment{Position: position, Speaker: segment.Speaker, Text: segment.Text, Categories: segment.Categories}
|
|
}
|