Collect optional validation context only when it can fit
This commit is contained in:
@@ -33,34 +33,58 @@ func prepareGlossaryContext(entries []glossary.Entry) glossaryContext {
|
||||
return glossaryContext{entries: entries, positions: positions, vocabulary: glossary.NewProtectedVocabulary(glossary.Vocabulary(entries, nil))}
|
||||
}
|
||||
|
||||
// validationContext supplies adjacent source segments within the current chunk
|
||||
// and glossary definitions mentioned in the candidate or that local context.
|
||||
// These fields travel in each item so batching accounts for their size.
|
||||
func (view glossaryContext) validationContext(segments []schema.Segment, position int, corrected string) ([]NeighboringSegment, []glossary.Entry) {
|
||||
var neighbors []NeighboringSegment
|
||||
source := segments[position].Text
|
||||
if position > 0 {
|
||||
neighbors = append(neighbors, neighboringSegment("preceding", segments[position-1]))
|
||||
}
|
||||
if position+1 < len(segments) {
|
||||
neighbors = append(neighbors, neighboringSegment("following", segments[position+1]))
|
||||
}
|
||||
type optionalValidationContext func(LLMValidationItem) ([]NeighboringSegment, []glossary.Entry)
|
||||
|
||||
texts := []string{source, corrected}
|
||||
for _, neighbor := range neighbors {
|
||||
texts = append(texts, neighbor.Text)
|
||||
// validationContextFor builds optional context on demand. Unchanged segment
|
||||
// matches are shared within the request. Corrected text is candidate-specific.
|
||||
func validationContextFor(req Request) optionalValidationContext {
|
||||
segments := req.sectionLookup().segments
|
||||
var view *glossaryContext
|
||||
if req.GlossaryView != nil {
|
||||
view = &req.GlossaryView.context
|
||||
}
|
||||
mentioned := view.vocabulary.MentionedIdentities(texts...)
|
||||
var matched []int
|
||||
for identity := range mentioned {
|
||||
matched = append(matched, view.positions[identity]...)
|
||||
var sourceMentions map[int]map[glossary.Identity]struct{}
|
||||
return func(item LLMValidationItem) ([]NeighboringSegment, []glossary.Entry) {
|
||||
position := item.segmentPosition
|
||||
var neighbors []NeighboringSegment
|
||||
if position > 0 {
|
||||
neighbors = append(neighbors, neighboringSegment("preceding", segments[position-1]))
|
||||
}
|
||||
if position+1 < len(segments) {
|
||||
neighbors = append(neighbors, neighboringSegment("following", segments[position+1]))
|
||||
}
|
||||
if view == nil {
|
||||
prepared := prepareGlossaryContext(req.Glossary)
|
||||
view = &prepared
|
||||
}
|
||||
if view.vocabulary.Empty() {
|
||||
return neighbors, nil
|
||||
}
|
||||
if sourceMentions == nil {
|
||||
sourceMentions = make(map[int]map[glossary.Identity]struct{})
|
||||
}
|
||||
mentioned := view.vocabulary.MentionedIdentities(item.CorrectedSegmentText)
|
||||
for i := max(0, position-1); i < min(len(segments), position+2); i++ {
|
||||
identities, ok := sourceMentions[i]
|
||||
if !ok {
|
||||
identities = view.vocabulary.MentionedIdentities(segments[i].Text)
|
||||
sourceMentions[i] = identities
|
||||
}
|
||||
for id := range identities {
|
||||
mentioned[id] = struct{}{}
|
||||
}
|
||||
}
|
||||
var matched []int
|
||||
for identity := range mentioned {
|
||||
matched = append(matched, view.positions[identity]...)
|
||||
}
|
||||
slices.Sort(matched)
|
||||
var entries []glossary.Entry
|
||||
for _, index := range matched {
|
||||
entries = append(entries, view.entries[index])
|
||||
}
|
||||
return neighbors, entries
|
||||
}
|
||||
slices.Sort(matched)
|
||||
var entries []glossary.Entry
|
||||
for _, index := range matched {
|
||||
entries = append(entries, view.entries[index])
|
||||
}
|
||||
return neighbors, entries
|
||||
}
|
||||
|
||||
func neighboringSegment(position string, segment schema.Segment) NeighboringSegment {
|
||||
|
||||
Reference in New Issue
Block a user