package validators import ( "slices" "gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking" "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" ) // GlossaryView holds the read-only matching data shared by a stage's validators // and apply-time checks. type GlossaryView struct { Protection ProtectedVocabulary context glossaryContext } func PrepareGlossaryView(entries []glossary.Entry, protected []glossary.Term) *GlossaryView { return &GlossaryView{Protection: NewProtectedVocabulary(protected), context: prepareGlossaryContext(entries)} } type glossaryContext struct { entries []glossary.Entry positions map[glossary.Identity][]int vocabulary ProtectedVocabulary } func prepareGlossaryContext(entries []glossary.Entry) glossaryContext { positions := make(map[glossary.Identity][]int, len(entries)) for i, entry := range entries { identity := glossary.Identity{Collection: entry.Collection, ID: entry.ID} positions[identity] = append(positions[identity], i) } return glossaryContext{entries: entries, positions: positions, vocabulary: NewProtectedVocabulary(glossary.Vocabulary(entries, nil))} } // validationContext supplies adjacent source segments within the current chunk // and glossary definitions mentioned in the candidate or that local context. // These fields travel in each item so batching accounts for their size. func (view glossaryContext) validationContext(segments []schema.Segment, position int, corrected string) ([]NeighboringSegment, []glossary.Entry) { var neighbors []NeighboringSegment source := segments[position].Text if position > 0 { neighbors = append(neighbors, neighboringSegment("preceding", segments[position-1])) } if position+1 < len(segments) { neighbors = append(neighbors, neighboringSegment("following", segments[position+1])) } texts := []string{source, corrected} for _, neighbor := range neighbors { texts = append(texts, neighbor.Text) } mentioned := make(map[glossary.Identity]struct{}) for _, text := range texts { for _, occurrence := range view.vocabulary.occurrences(text) { mentioned[occurrence.identity] = struct{}{} } } var matched []int for identity := range mentioned { matched = append(matched, view.positions[identity]...) } slices.Sort(matched) var entries []glossary.Entry for _, index := range matched { entries = append(entries, view.entries[index]) } return neighbors, entries } // fitValidationContext drops optional context before treating required candidate // text as oversized. Glossary definitions take priority over adjacent segments. func fitValidationContext(item LLMValidationItem, budget int, estimator chunking.TokenEstimator) LLMValidationItem { for { tokens, err := estimateBatchTokens(estimator, []LLMValidationItem{item}) if err != nil || tokens <= budget { return item } if len(item.NeighboringSegments) > 0 { item.NeighboringSegments = item.NeighboringSegments[:len(item.NeighboringSegments)-1] } else if len(item.GlossaryEntries) > 0 { item.GlossaryEntries = item.GlossaryEntries[:len(item.GlossaryEntries)-1] } else { return item } } } func neighboringSegment(position string, segment schema.Segment) NeighboringSegment { return NeighboringSegment{Position: position, Speaker: segment.Speaker, Text: segment.Text, Categories: segment.Categories} }