Files
audita/internal/framework/validators/llm_batching.go

172 lines
5.0 KiB
Go
Raw Normal View History

2026-05-12 01:25:52 +00:00
package validators
import (
"encoding/json"
"fmt"
"slices"
2026-05-12 01:25:52 +00:00
2026-10-11 01:40:38 +00:00
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
2026-05-12 01:25:52 +00:00
)
type LLMValidationBatch struct {
BatchIndex int `json:"batch_index"`
Items []LLMValidationItem `json:"items"`
TokenCount int `json:"token_count"`
}
func prepareValidationBatches(items []LLMValidationItem, budget int, estimator chunking.TokenEstimator) ([]LLMValidationBatch, []Decision, error) {
if estimator == nil {
estimator = chunking.NewSimpleTokenEstimator()
}
simple, _ := estimator.(*chunking.SimpleTokenEstimator)
usable := make([]LLMValidationItem, 0, len(items))
var costs []validationCost
var sizes []int
var oversized []Decision
for _, item := range items {
fitted, err := fitValidationContext(item, budget, estimator)
if err != nil {
return nil, oversized, err
}
if fitted.tokens > budget {
oversized = append(oversized, oversizedValidationDecision(item.CorrectionIndex))
continue
}
usable = append(usable, fitted.item)
if simple != nil {
costs = append(costs, fitted.cost)
} else {
sizes = append(sizes, fitted.tokens)
}
}
if simple != nil {
batches, err := chunkSimpleValidationItems(usable, costs, budget, simple)
return batches, oversized, err
}
batches, err := chunkSerializedValidationItems(usable, sizes, budget, estimator)
return batches, oversized, err
}
2026-05-12 01:25:52 +00:00
func ChunkLLMValidationItems(items []LLMValidationItem, maxPromptTokens int, estimator chunking.TokenEstimator) ([]LLMValidationBatch, error) {
if maxPromptTokens <= 0 {
return nil, fmt.Errorf("validation max prompt tokens must be greater than zero")
}
if len(items) == 0 {
return nil, fmt.Errorf("validation input must contain at least one proposal")
}
if estimator == nil {
estimator = chunking.NewSimpleTokenEstimator()
}
if simple, ok := estimator.(*chunking.SimpleTokenEstimator); ok {
return chunkSimpleValidationItems(items, nil, maxPromptTokens, simple)
}
2026-05-12 01:25:52 +00:00
return chunkSerializedValidationItems(items, nil, maxPromptTokens, estimator)
}
func chunkSerializedValidationItems(items []LLMValidationItem, sizes []int, maxPromptTokens int, estimator chunking.TokenEstimator) ([]LLMValidationBatch, error) {
var batches []LLMValidationBatch
var current []LLMValidationItem
2026-05-12 01:25:52 +00:00
currentTokens := 0
for i, item := range items {
var singleTokens int
if sizes != nil {
singleTokens = sizes[i]
} else {
var err error
singleTokens, err = estimateBatchTokens(estimator, []LLMValidationItem{item})
if err != nil {
return nil, err
}
2026-05-12 01:25:52 +00:00
}
if singleTokens > maxPromptTokens {
return nil, fmt.Errorf("single validation proposal exceeds max prompt tokens")
}
if len(current) == 0 {
current, currentTokens = []LLMValidationItem{item}, singleTokens
continue
}
2026-05-12 01:25:52 +00:00
candidate := append(append([]LLMValidationItem(nil), current...), item)
candidateTokens, err := estimateBatchTokens(estimator, candidate)
if err != nil {
return nil, err
}
if candidateTokens > maxPromptTokens {
2026-05-12 01:25:52 +00:00
batches = append(batches, LLMValidationBatch{
BatchIndex: len(batches),
Items: append([]LLMValidationItem(nil), current...),
TokenCount: currentTokens,
})
current = []LLMValidationItem{item}
currentTokens = singleTokens
continue
}
current = candidate
currentTokens = candidateTokens
}
if len(current) > 0 {
batches = append(batches, LLMValidationBatch{
BatchIndex: len(batches),
Items: append([]LLMValidationItem(nil), current...),
TokenCount: currentTokens,
})
}
return batches, nil
}
func chunkSimpleValidationItems(items []LLMValidationItem, costs []validationCost, budget int, estimator *chunking.SimpleTokenEstimator) ([]LLMValidationBatch, error) {
var batches []LLMValidationBatch
start := 0
var batchCost validationCost
for i, item := range items {
var cost validationCost
if costs != nil {
cost = costs[i]
} else {
var err error
cost, err = measureValidationItem(estimator, item)
if err != nil {
return nil, err
}
}
if cost.arrayTokens() > budget {
return nil, fmt.Errorf("single validation proposal exceeds max prompt tokens")
}
candidateCost := batchCost.add(cost)
if i > start {
candidateCost.punctuation++ // separator between items
if candidateCost.arrayTokens() > budget {
batches = append(batches, LLMValidationBatch{
BatchIndex: len(batches),
Items: slices.Clone(items[start:i]),
TokenCount: batchCost.arrayTokens(),
})
start, candidateCost = i, cost
}
}
batchCost = candidateCost
}
if start < len(items) {
batches = append(batches, LLMValidationBatch{
BatchIndex: len(batches),
Items: slices.Clone(items[start:]),
TokenCount: batchCost.arrayTokens(),
})
}
return batches, nil
}
2026-05-12 01:25:52 +00:00
func estimateBatchTokens(estimator chunking.TokenEstimator, batch []LLMValidationItem) (int, error) {
payload, err := json.Marshal(localValidationItems(batch))
2026-05-12 01:25:52 +00:00
if err != nil {
return 0, fmt.Errorf("marshal batch payload: %w", err)
}
return estimator.EstimateTokens(string(payload)), nil
}