Measure only the usable validation context prefix
This commit is contained in:
@@ -15,35 +15,35 @@ type LLMValidationBatch struct {
|
||||
}
|
||||
|
||||
func prepareValidationBatches(items []LLMValidationItem, budget int, estimator chunking.TokenEstimator) ([]LLMValidationBatch, []Decision, error) {
|
||||
if simple, ok := estimator.(*chunking.SimpleTokenEstimator); ok {
|
||||
usable := make([]LLMValidationItem, 0, len(items))
|
||||
costs := make([]validationCost, 0, len(items))
|
||||
var oversized []Decision
|
||||
for _, item := range items {
|
||||
fitted, cost, err := fitSimpleValidationContext(item, budget, simple)
|
||||
if err != nil {
|
||||
return nil, oversized, err
|
||||
}
|
||||
if cost.arrayTokens() > budget {
|
||||
oversized = append(oversized, oversizedValidationDecision(item.CorrectionIndex))
|
||||
} else {
|
||||
usable = append(usable, fitted)
|
||||
costs = append(costs, cost)
|
||||
}
|
||||
if estimator == nil {
|
||||
estimator = chunking.NewSimpleTokenEstimator()
|
||||
}
|
||||
simple, _ := estimator.(*chunking.SimpleTokenEstimator)
|
||||
usable := make([]LLMValidationItem, 0, len(items))
|
||||
var costs []validationCost
|
||||
var sizes []int
|
||||
var oversized []Decision
|
||||
for _, item := range items {
|
||||
fitted, err := fitValidationContext(item, budget, estimator)
|
||||
if err != nil {
|
||||
return nil, oversized, err
|
||||
}
|
||||
if fitted.tokens > budget {
|
||||
oversized = append(oversized, oversizedValidationDecision(item.CorrectionIndex))
|
||||
continue
|
||||
}
|
||||
usable = append(usable, fitted.item)
|
||||
if simple != nil {
|
||||
costs = append(costs, fitted.cost)
|
||||
} else {
|
||||
sizes = append(sizes, fitted.tokens)
|
||||
}
|
||||
}
|
||||
if simple != nil {
|
||||
batches, err := chunkSimpleValidationItems(usable, costs, budget, simple)
|
||||
return batches, oversized, err
|
||||
}
|
||||
|
||||
for i := range items {
|
||||
items[i] = fitValidationContext(items[i], budget, estimator)
|
||||
}
|
||||
oversized := oversizedValidationDecisions(items, budget, estimator)
|
||||
usable := filterItemsByDecision(items, oversized)
|
||||
if len(usable) == 0 {
|
||||
return nil, oversized, nil
|
||||
}
|
||||
batches, err := ChunkLLMValidationItems(usable, budget, estimator)
|
||||
batches, err := chunkSerializedValidationItems(usable, sizes, budget, estimator)
|
||||
return batches, oversized, err
|
||||
}
|
||||
|
||||
@@ -61,26 +61,40 @@ func ChunkLLMValidationItems(items []LLMValidationItem, maxPromptTokens int, est
|
||||
return chunkSimpleValidationItems(items, nil, maxPromptTokens, simple)
|
||||
}
|
||||
|
||||
batches := make([]LLMValidationBatch, 0)
|
||||
current := make([]LLMValidationItem, 0)
|
||||
return chunkSerializedValidationItems(items, nil, maxPromptTokens, estimator)
|
||||
}
|
||||
|
||||
func chunkSerializedValidationItems(items []LLMValidationItem, sizes []int, maxPromptTokens int, estimator chunking.TokenEstimator) ([]LLMValidationBatch, error) {
|
||||
var batches []LLMValidationBatch
|
||||
var current []LLMValidationItem
|
||||
currentTokens := 0
|
||||
|
||||
for _, item := range items {
|
||||
singleTokens, err := estimateBatchTokens(estimator, []LLMValidationItem{item})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
for i, item := range items {
|
||||
var singleTokens int
|
||||
if sizes != nil {
|
||||
singleTokens = sizes[i]
|
||||
} else {
|
||||
var err error
|
||||
singleTokens, err = estimateBatchTokens(estimator, []LLMValidationItem{item})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if singleTokens > maxPromptTokens {
|
||||
return nil, fmt.Errorf("single validation proposal exceeds max prompt tokens")
|
||||
}
|
||||
|
||||
if len(current) == 0 {
|
||||
current, currentTokens = []LLMValidationItem{item}, singleTokens
|
||||
continue
|
||||
}
|
||||
candidate := append(append([]LLMValidationItem(nil), current...), item)
|
||||
candidateTokens, err := estimateBatchTokens(estimator, candidate)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(current) > 0 && candidateTokens > maxPromptTokens {
|
||||
if candidateTokens > maxPromptTokens {
|
||||
batches = append(batches, LLMValidationBatch{
|
||||
BatchIndex: len(batches),
|
||||
Items: append([]LLMValidationItem(nil), current...),
|
||||
|
||||
Reference in New Issue
Block a user