Measure only the usable validation context prefix

This commit is contained in:
2026-11-16 14:14:21 +00:00
parent 0c144fd5a5
commit b1d1b8302c
8 changed files with 229 additions and 144 deletions

View File

@@ -15,35 +15,35 @@ type LLMValidationBatch struct {
}
func prepareValidationBatches(items []LLMValidationItem, budget int, estimator chunking.TokenEstimator) ([]LLMValidationBatch, []Decision, error) {
if simple, ok := estimator.(*chunking.SimpleTokenEstimator); ok {
usable := make([]LLMValidationItem, 0, len(items))
costs := make([]validationCost, 0, len(items))
var oversized []Decision
for _, item := range items {
fitted, cost, err := fitSimpleValidationContext(item, budget, simple)
if err != nil {
return nil, oversized, err
}
if cost.arrayTokens() > budget {
oversized = append(oversized, oversizedValidationDecision(item.CorrectionIndex))
} else {
usable = append(usable, fitted)
costs = append(costs, cost)
}
if estimator == nil {
estimator = chunking.NewSimpleTokenEstimator()
}
simple, _ := estimator.(*chunking.SimpleTokenEstimator)
usable := make([]LLMValidationItem, 0, len(items))
var costs []validationCost
var sizes []int
var oversized []Decision
for _, item := range items {
fitted, err := fitValidationContext(item, budget, estimator)
if err != nil {
return nil, oversized, err
}
if fitted.tokens > budget {
oversized = append(oversized, oversizedValidationDecision(item.CorrectionIndex))
continue
}
usable = append(usable, fitted.item)
if simple != nil {
costs = append(costs, fitted.cost)
} else {
sizes = append(sizes, fitted.tokens)
}
}
if simple != nil {
batches, err := chunkSimpleValidationItems(usable, costs, budget, simple)
return batches, oversized, err
}
for i := range items {
items[i] = fitValidationContext(items[i], budget, estimator)
}
oversized := oversizedValidationDecisions(items, budget, estimator)
usable := filterItemsByDecision(items, oversized)
if len(usable) == 0 {
return nil, oversized, nil
}
batches, err := ChunkLLMValidationItems(usable, budget, estimator)
batches, err := chunkSerializedValidationItems(usable, sizes, budget, estimator)
return batches, oversized, err
}
@@ -61,26 +61,40 @@ func ChunkLLMValidationItems(items []LLMValidationItem, maxPromptTokens int, est
return chunkSimpleValidationItems(items, nil, maxPromptTokens, simple)
}
batches := make([]LLMValidationBatch, 0)
current := make([]LLMValidationItem, 0)
return chunkSerializedValidationItems(items, nil, maxPromptTokens, estimator)
}
func chunkSerializedValidationItems(items []LLMValidationItem, sizes []int, maxPromptTokens int, estimator chunking.TokenEstimator) ([]LLMValidationBatch, error) {
var batches []LLMValidationBatch
var current []LLMValidationItem
currentTokens := 0
for _, item := range items {
singleTokens, err := estimateBatchTokens(estimator, []LLMValidationItem{item})
if err != nil {
return nil, err
for i, item := range items {
var singleTokens int
if sizes != nil {
singleTokens = sizes[i]
} else {
var err error
singleTokens, err = estimateBatchTokens(estimator, []LLMValidationItem{item})
if err != nil {
return nil, err
}
}
if singleTokens > maxPromptTokens {
return nil, fmt.Errorf("single validation proposal exceeds max prompt tokens")
}
if len(current) == 0 {
current, currentTokens = []LLMValidationItem{item}, singleTokens
continue
}
candidate := append(append([]LLMValidationItem(nil), current...), item)
candidateTokens, err := estimateBatchTokens(estimator, candidate)
if err != nil {
return nil, err
}
if len(current) > 0 && candidateTokens > maxPromptTokens {
if candidateTokens > maxPromptTokens {
batches = append(batches, LLMValidationBatch{
BatchIndex: len(batches),
Items: append([]LLMValidationItem(nil), current...),