Reuse validation costs when fitting and batching
This commit is contained in:
90
internal/framework/validators/llm_budget.go
Normal file
90
internal/framework/validators/llm_budget.go
Normal file
@@ -0,0 +1,90 @@
|
||||
package validators
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
|
||||
)
|
||||
|
||||
// Compact JSON fragments in this budget begin and end with non-whitespace.
|
||||
// When fragments join, their touching word runs merge. Counting only extra
|
||||
// runs inside each fragment makes costs additive, with one run for the whole
|
||||
// payload and punctuation rounded only after assembling the array.
|
||||
type validationCost struct {
|
||||
extraRuns int
|
||||
punctuation int
|
||||
}
|
||||
|
||||
func (c validationCost) add(other validationCost) validationCost {
|
||||
return validationCost{c.extraRuns + other.extraRuns, c.punctuation + other.punctuation}
|
||||
}
|
||||
|
||||
func (c validationCost) arrayTokens() int {
|
||||
return 1 + c.extraRuns + (c.punctuation+2)/2 // opening and closing brackets
|
||||
}
|
||||
|
||||
func measureJSON(estimator *chunking.SimpleTokenEstimator, fragment []byte) validationCost {
|
||||
words, punctuation := estimator.CountComponents(string(fragment))
|
||||
return validationCost{words - 1, punctuation}
|
||||
}
|
||||
|
||||
func measureValidationItem(estimator *chunking.SimpleTokenEstimator, item LLMValidationItem) (validationCost, error) {
|
||||
// Every positive local reference has the same cost, regardless of digit count.
|
||||
item.CorrectionIndex = 1
|
||||
payload, err := json.Marshal(item)
|
||||
if err != nil {
|
||||
return validationCost{}, fmt.Errorf("marshal batch payload: %w", err)
|
||||
}
|
||||
return measureJSON(estimator, payload), nil
|
||||
}
|
||||
|
||||
func measureContextPrefixes[T any](estimator *chunking.SimpleTokenEstimator, field string, entries []T) ([]validationCost, error) {
|
||||
prefixes := make([]validationCost, len(entries)+1)
|
||||
cost := measureJSON(estimator, []byte(`,"`+field+`":[]`))
|
||||
for i, entry := range entries {
|
||||
payload, err := json.Marshal(entry)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal validation %s: %w", field, err)
|
||||
}
|
||||
if i > 0 {
|
||||
cost.punctuation++ // separator between entries
|
||||
}
|
||||
cost = cost.add(measureJSON(estimator, payload))
|
||||
prefixes[i+1] = cost
|
||||
}
|
||||
return prefixes, nil
|
||||
}
|
||||
|
||||
func fitSimpleValidationContext(item LLMValidationItem, budget int, estimator *chunking.SimpleTokenEstimator) (LLMValidationItem, validationCost, error) {
|
||||
required := item
|
||||
required.NeighboringSegments, required.GlossaryEntries = nil, nil
|
||||
base, err := measureValidationItem(estimator, required)
|
||||
if err != nil {
|
||||
return item, validationCost{}, err
|
||||
}
|
||||
if len(item.NeighboringSegments)+len(item.GlossaryEntries) == 0 {
|
||||
return item, base, nil
|
||||
}
|
||||
neighbors, err := measureContextPrefixes(estimator, "neighboring_segments", item.NeighboringSegments)
|
||||
if err != nil {
|
||||
return item, validationCost{}, err
|
||||
}
|
||||
glossary, err := measureContextPrefixes(estimator, "glossary_entries", item.GlossaryEntries)
|
||||
if err != nil {
|
||||
return item, validationCost{}, err
|
||||
}
|
||||
n, g := len(item.NeighboringSegments), len(item.GlossaryEntries)
|
||||
cost := base.add(neighbors[n]).add(glossary[g])
|
||||
for n > 0 && cost.arrayTokens() > budget {
|
||||
n--
|
||||
cost = base.add(neighbors[n]).add(glossary[g])
|
||||
}
|
||||
for g > 0 && cost.arrayTokens() > budget {
|
||||
g--
|
||||
cost = base.add(glossary[g])
|
||||
}
|
||||
item.NeighboringSegments = item.NeighboringSegments[:n]
|
||||
item.GlossaryEntries = item.GlossaryEntries[:g]
|
||||
return item, cost, nil
|
||||
}
|
||||
Reference in New Issue
Block a user