Reuse validation costs when fitting and batching

internal/core/chunking/tokens.go

@@ -1,7 +1,6 @@
 package chunking
 
 import (
-	"strings"
 	"unicode"
 )
 
@@ -24,24 +23,24 @@ func NewSimpleTokenEstimator() *SimpleTokenEstimator {
 // This implementation uses a simple heuristic: count words and punctuation as tokens.
 // The estimate is deterministic and stable for the same input text.
 func (e *SimpleTokenEstimator) EstimateTokens(text string) int {
-	if text == "" {
-		return 0
-	}
+	words, punctuation := e.CountComponents(text)
+	return words + punctuation/2
+}
 
-	// Simple heuristic: split on whitespace and count non-empty segments
-	words := strings.Fields(text)
-	tokenCount := len(words)
-
-	// Add some estimate for punctuation that might be separate tokens
-	punctuationCount := 0
+// CountComponents returns unrounded counts for budgets composed from fragments.
+func (e *SimpleTokenEstimator) CountComponents(text string) (words, punctuation int) {
+	inWord := false
 	for _, r := range text {
+		nonspace := !unicode.IsSpace(r)
+		if nonspace && !inWord {
+			words++
+		}
+		inWord = nonspace
 		if unicode.IsPunct(r) && r != '\'' && r != '-' && r != '_' {
-			punctuationCount++
+			punctuation++
 		}
 	}
-
-	// Rough estimate: each word is a token, plus half the punctuation as separate tokens
-	return tokenCount + (punctuationCount / 2)
+	return words, punctuation
 }
 
 // ConstTokenEstimator returns a constant token count for testing purposes.