Normalize nested intervals and build merges incrementally

This commit is contained in:
2026-11-16 03:59:44 +00:00
parent f465496dd8
commit 27f494d4e4
6 changed files with 173 additions and 229 deletions

View File

@@ -23,24 +23,39 @@ func NewSimpleTokenEstimator() *SimpleTokenEstimator {
// This implementation uses a simple heuristic: count words and punctuation as tokens.
// The estimate is deterministic and stable for the same input text.
func (e *SimpleTokenEstimator) EstimateTokens(text string) int {
words, punctuation := e.CountComponents(text)
return words + punctuation/2
var count TokenCount
count.Add(text)
return count.Estimate()
}
// CountComponents returns unrounded counts for budgets composed from fragments.
func (e *SimpleTokenEstimator) CountComponents(text string) (words, punctuation int) {
inWord := false
var count TokenCount
count.Add(text)
return count.words, count.punctuation
}
// TokenCount can be copied and extended with adjacent text fragments.
type TokenCount struct {
words, punctuation int
inWord bool
}
func (c *TokenCount) Add(text string) {
for _, r := range text {
nonspace := !unicode.IsSpace(r)
if nonspace && !inWord {
words++
if nonspace && !c.inWord {
c.words++
}
inWord = nonspace
c.inWord = nonspace
if unicode.IsPunct(r) && r != '\'' && r != '-' && r != '_' {
punctuation++
c.punctuation++
}
}
return words, punctuation
}
func (c TokenCount) Estimate() int {
return c.words + c.punctuation/2
}
// ConstTokenEstimator returns a constant token count for testing purposes.