package validators import ( "encoding/json" "fmt" "slices" "gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking" ) type LLMValidationBatch struct { BatchIndex int `json:"batch_index"` Items []LLMValidationItem `json:"items"` TokenCount int `json:"token_count"` } func prepareValidationBatches(items []LLMValidationItem, budget int, estimator chunking.TokenEstimator, context optionalValidationContext) ([]LLMValidationBatch, []Decision, error) { if estimator == nil { estimator = chunking.NewSimpleTokenEstimator() } simple, _ := estimator.(*chunking.SimpleTokenEstimator) usable := make([]LLMValidationItem, 0, len(items)) var costs []validationCost var sizes []int var oversized []Decision for _, item := range items { fitted, err := fitValidationContext(item, budget, estimator, context) if err != nil { return nil, oversized, err } if fitted.tokens > budget { oversized = append(oversized, oversizedValidationDecision(item.CorrectionIndex)) continue } usable = append(usable, fitted.item) if simple != nil { costs = append(costs, fitted.cost) } else { sizes = append(sizes, fitted.tokens) } } if simple != nil { batches, err := chunkSimpleValidationItems(usable, costs, budget, simple) return batches, oversized, err } batches, err := chunkSerializedValidationItems(usable, sizes, budget, estimator) return batches, oversized, err } func ChunkLLMValidationItems(items []LLMValidationItem, maxPromptTokens int, estimator chunking.TokenEstimator) ([]LLMValidationBatch, error) { if maxPromptTokens <= 0 { return nil, fmt.Errorf("validation max prompt tokens must be greater than zero") } if len(items) == 0 { return nil, fmt.Errorf("validation input must contain at least one proposal") } if estimator == nil { estimator = chunking.NewSimpleTokenEstimator() } if simple, ok := estimator.(*chunking.SimpleTokenEstimator); ok { return chunkSimpleValidationItems(items, nil, maxPromptTokens, simple) } return chunkSerializedValidationItems(items, nil, maxPromptTokens, estimator) } func chunkSerializedValidationItems(items []LLMValidationItem, sizes []int, maxPromptTokens int, estimator chunking.TokenEstimator) ([]LLMValidationBatch, error) { var batches []LLMValidationBatch var current []LLMValidationItem currentTokens := 0 for i, item := range items { var singleTokens int if sizes != nil { singleTokens = sizes[i] } else { var err error singleTokens, err = estimateBatchTokens(estimator, []LLMValidationItem{item}) if err != nil { return nil, err } } if singleTokens > maxPromptTokens { return nil, fmt.Errorf("single validation proposal exceeds max prompt tokens") } if len(current) == 0 { current, currentTokens = []LLMValidationItem{item}, singleTokens continue } candidate := append(append([]LLMValidationItem(nil), current...), item) candidateTokens, err := estimateBatchTokens(estimator, candidate) if err != nil { return nil, err } if candidateTokens > maxPromptTokens { batches = append(batches, LLMValidationBatch{ BatchIndex: len(batches), Items: append([]LLMValidationItem(nil), current...), TokenCount: currentTokens, }) current = []LLMValidationItem{item} currentTokens = singleTokens continue } current = candidate currentTokens = candidateTokens } if len(current) > 0 { batches = append(batches, LLMValidationBatch{ BatchIndex: len(batches), Items: append([]LLMValidationItem(nil), current...), TokenCount: currentTokens, }) } return batches, nil } func chunkSimpleValidationItems(items []LLMValidationItem, costs []validationCost, budget int, estimator *chunking.SimpleTokenEstimator) ([]LLMValidationBatch, error) { var batches []LLMValidationBatch start := 0 var batchCost validationCost for i, item := range items { var cost validationCost if costs != nil { cost = costs[i] } else { var err error cost, err = measureValidationItem(estimator, item) if err != nil { return nil, err } } if cost.arrayTokens() > budget { return nil, fmt.Errorf("single validation proposal exceeds max prompt tokens") } candidateCost := batchCost.add(cost) if i > start { candidateCost.punctuation++ // separator between items if candidateCost.arrayTokens() > budget { batches = append(batches, LLMValidationBatch{ BatchIndex: len(batches), Items: slices.Clone(items[start:i]), TokenCount: batchCost.arrayTokens(), }) start, candidateCost = i, cost } } batchCost = candidateCost } if start < len(items) { batches = append(batches, LLMValidationBatch{ BatchIndex: len(batches), Items: slices.Clone(items[start:]), TokenCount: batchCost.arrayTokens(), }) } return batches, nil } func estimateBatchTokens(estimator chunking.TokenEstimator, batch []LLMValidationItem) (int, error) { payload, err := json.Marshal(localValidationItems(batch)) if err != nil { return 0, fmt.Errorf("marshal batch payload: %w", err) } return estimator.EstimateTokens(string(payload)), nil }