Normalize nested intervals and build merges incrementally
This commit is contained in:
@@ -23,24 +23,39 @@ func NewSimpleTokenEstimator() *SimpleTokenEstimator {
|
||||
// This implementation uses a simple heuristic: count words and punctuation as tokens.
|
||||
// The estimate is deterministic and stable for the same input text.
|
||||
func (e *SimpleTokenEstimator) EstimateTokens(text string) int {
|
||||
words, punctuation := e.CountComponents(text)
|
||||
return words + punctuation/2
|
||||
var count TokenCount
|
||||
count.Add(text)
|
||||
return count.Estimate()
|
||||
}
|
||||
|
||||
// CountComponents returns unrounded counts for budgets composed from fragments.
|
||||
func (e *SimpleTokenEstimator) CountComponents(text string) (words, punctuation int) {
|
||||
inWord := false
|
||||
var count TokenCount
|
||||
count.Add(text)
|
||||
return count.words, count.punctuation
|
||||
}
|
||||
|
||||
// TokenCount can be copied and extended with adjacent text fragments.
|
||||
type TokenCount struct {
|
||||
words, punctuation int
|
||||
inWord bool
|
||||
}
|
||||
|
||||
func (c *TokenCount) Add(text string) {
|
||||
for _, r := range text {
|
||||
nonspace := !unicode.IsSpace(r)
|
||||
if nonspace && !inWord {
|
||||
words++
|
||||
if nonspace && !c.inWord {
|
||||
c.words++
|
||||
}
|
||||
inWord = nonspace
|
||||
c.inWord = nonspace
|
||||
if unicode.IsPunct(r) && r != '\'' && r != '-' && r != '_' {
|
||||
punctuation++
|
||||
c.punctuation++
|
||||
}
|
||||
}
|
||||
return words, punctuation
|
||||
}
|
||||
|
||||
func (c TokenCount) Estimate() int {
|
||||
return c.words + c.punctuation/2
|
||||
}
|
||||
|
||||
// ConstTokenEstimator returns a constant token count for testing purposes.
|
||||
|
||||
Reference in New Issue
Block a user