Files
audita/internal/core/chunking/tokens.go

70 lines
1.9 KiB
Go

package chunking
import (
"unicode"
)
// TokenEstimator provides a deterministic token estimation suitable for prompt budgeting.
// The estimator is approximate but stable, isolated, and replaceable.
type TokenEstimator interface {
EstimateTokens(text string) int
}
// SimpleTokenEstimator provides a basic deterministic token estimation.
// This uses a simple heuristic based on word count and punctuation.
type SimpleTokenEstimator struct{}
// NewSimpleTokenEstimator creates a new simple token estimator.
func NewSimpleTokenEstimator() *SimpleTokenEstimator {
return &SimpleTokenEstimator{}
}
// EstimateTokens provides a rough estimate of the number of tokens in the given text.
// This implementation uses a simple heuristic: count words and punctuation as tokens.
// The estimate is deterministic and stable for the same input text.
func (e *SimpleTokenEstimator) EstimateTokens(text string) int {
var count TokenCount
count.Add(text)
return count.Estimate()
}
// CountComponents returns unrounded counts for budgets composed from fragments.
func (e *SimpleTokenEstimator) CountComponents(text string) (words, punctuation int) {
var count TokenCount
count.Add(text)
return count.words, count.punctuation
}
// TokenCount can be copied and extended with adjacent text fragments.
type TokenCount struct {
words, punctuation int
inWord bool
}
func (c *TokenCount) Add(text string) {
for _, r := range text {
nonspace := !unicode.IsSpace(r)
if nonspace && !c.inWord {
c.words++
}
c.inWord = nonspace
if unicode.IsPunct(r) && r != '\'' && r != '-' && r != '_' {
c.punctuation++
}
}
}
func (c TokenCount) Estimate() int {
return c.words + c.punctuation/2
}
// ConstTokenEstimator returns a constant token count for testing purposes.
type ConstTokenEstimator struct {
Tokens int
}
// EstimateTokens returns the configured constant token count.
func (e *ConstTokenEstimator) EstimateTokens(text string) int {
return e.Tokens
}