Measure only the usable validation context prefix
This commit is contained in:
@@ -39,52 +39,70 @@ func measureValidationItem(estimator *chunking.SimpleTokenEstimator, item LLMVal
|
||||
return measureJSON(estimator, payload), nil
|
||||
}
|
||||
|
||||
func measureContextPrefixes[T any](estimator *chunking.SimpleTokenEstimator, field string, entries []T) ([]validationCost, error) {
|
||||
prefixes := make([]validationCost, len(entries)+1)
|
||||
cost := measureJSON(estimator, []byte(`,"`+field+`":[]`))
|
||||
for i, entry := range entries {
|
||||
payload, err := json.Marshal(entry)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal validation %s: %w", field, err)
|
||||
}
|
||||
if i > 0 {
|
||||
cost.punctuation++ // separator between entries
|
||||
}
|
||||
cost = cost.add(measureJSON(estimator, payload))
|
||||
prefixes[i+1] = cost
|
||||
}
|
||||
return prefixes, nil
|
||||
type fittedValidationItem struct {
|
||||
item LLMValidationItem
|
||||
cost validationCost
|
||||
tokens int
|
||||
}
|
||||
|
||||
func fitSimpleValidationContext(item LLMValidationItem, budget int, estimator *chunking.SimpleTokenEstimator) (LLMValidationItem, validationCost, error) {
|
||||
required := item
|
||||
required.NeighboringSegments, required.GlossaryEntries = nil, nil
|
||||
base, err := measureValidationItem(estimator, required)
|
||||
if err != nil {
|
||||
return item, validationCost{}, err
|
||||
// fitValidationContext tries the required payload followed by glossary entries
|
||||
// and then neighboring segments. The first excess ends the usable prefix.
|
||||
func fitValidationContext(item LLMValidationItem, budget int, estimator chunking.TokenEstimator) (fittedValidationItem, error) {
|
||||
fitted := fittedValidationItem{item: item}
|
||||
fitted.item.GlossaryEntries = item.GlossaryEntries[:0]
|
||||
fitted.item.NeighboringSegments = item.NeighboringSegments[:0]
|
||||
simple, _ := estimator.(*chunking.SimpleTokenEstimator)
|
||||
var err error
|
||||
if simple != nil {
|
||||
fitted.cost, err = measureValidationItem(simple, fitted.item)
|
||||
fitted.tokens = fitted.cost.arrayTokens()
|
||||
} else {
|
||||
fitted.tokens, err = estimateBatchTokens(estimator, []LLMValidationItem{fitted.item})
|
||||
}
|
||||
if len(item.NeighboringSegments)+len(item.GlossaryEntries) == 0 {
|
||||
return item, base, nil
|
||||
if err != nil || fitted.tokens > budget {
|
||||
return fitted, err
|
||||
}
|
||||
neighbors, err := measureContextPrefixes(estimator, "neighboring_segments", item.NeighboringSegments)
|
||||
if err != nil {
|
||||
return item, validationCost{}, err
|
||||
try := func(candidate LLMValidationItem, field string, first bool, entry any) (bool, error) {
|
||||
cost := fitted.cost
|
||||
var tokens int
|
||||
if simple != nil {
|
||||
payload, err := json.Marshal(entry)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("marshal validation %s: %w", field, err)
|
||||
}
|
||||
if first {
|
||||
cost = cost.add(measureJSON(simple, []byte(`,"`+field+`":[]`)))
|
||||
} else {
|
||||
cost.punctuation++ // separator between entries
|
||||
}
|
||||
cost = cost.add(measureJSON(simple, payload))
|
||||
tokens = cost.arrayTokens()
|
||||
} else {
|
||||
var err error
|
||||
tokens, err = estimateBatchTokens(estimator, []LLMValidationItem{candidate})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
if tokens > budget {
|
||||
return false, nil
|
||||
}
|
||||
fitted = fittedValidationItem{item: candidate, cost: cost, tokens: tokens}
|
||||
return true, nil
|
||||
}
|
||||
glossary, err := measureContextPrefixes(estimator, "glossary_entries", item.GlossaryEntries)
|
||||
if err != nil {
|
||||
return item, validationCost{}, err
|
||||
for i, entry := range item.GlossaryEntries {
|
||||
candidate := fitted.item
|
||||
candidate.GlossaryEntries = item.GlossaryEntries[:i+1]
|
||||
if ok, err := try(candidate, "glossary_entries", i == 0, entry); !ok {
|
||||
return fitted, err
|
||||
}
|
||||
}
|
||||
n, g := len(item.NeighboringSegments), len(item.GlossaryEntries)
|
||||
cost := base.add(neighbors[n]).add(glossary[g])
|
||||
for n > 0 && cost.arrayTokens() > budget {
|
||||
n--
|
||||
cost = base.add(neighbors[n]).add(glossary[g])
|
||||
for i, entry := range item.NeighboringSegments {
|
||||
candidate := fitted.item
|
||||
candidate.NeighboringSegments = item.NeighboringSegments[:i+1]
|
||||
if ok, err := try(candidate, "neighboring_segments", i == 0, entry); !ok {
|
||||
return fitted, err
|
||||
}
|
||||
}
|
||||
for g > 0 && cost.arrayTokens() > budget {
|
||||
g--
|
||||
cost = base.add(glossary[g])
|
||||
}
|
||||
item.NeighboringSegments = item.NeighboringSegments[:n]
|
||||
item.GlossaryEntries = item.GlossaryEntries[:g]
|
||||
return item, cost, nil
|
||||
return fitted, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user