Measure only the usable validation context prefix

This commit is contained in:
2026-11-16 14:14:21 +00:00
parent 0c144fd5a5
commit b1d1b8302c
8 changed files with 229 additions and 144 deletions

View File

@@ -15,35 +15,35 @@ type LLMValidationBatch struct {
}
func prepareValidationBatches(items []LLMValidationItem, budget int, estimator chunking.TokenEstimator) ([]LLMValidationBatch, []Decision, error) {
if simple, ok := estimator.(*chunking.SimpleTokenEstimator); ok {
usable := make([]LLMValidationItem, 0, len(items))
costs := make([]validationCost, 0, len(items))
var oversized []Decision
for _, item := range items {
fitted, cost, err := fitSimpleValidationContext(item, budget, simple)
if err != nil {
return nil, oversized, err
}
if cost.arrayTokens() > budget {
oversized = append(oversized, oversizedValidationDecision(item.CorrectionIndex))
} else {
usable = append(usable, fitted)
costs = append(costs, cost)
}
if estimator == nil {
estimator = chunking.NewSimpleTokenEstimator()
}
simple, _ := estimator.(*chunking.SimpleTokenEstimator)
usable := make([]LLMValidationItem, 0, len(items))
var costs []validationCost
var sizes []int
var oversized []Decision
for _, item := range items {
fitted, err := fitValidationContext(item, budget, estimator)
if err != nil {
return nil, oversized, err
}
if fitted.tokens > budget {
oversized = append(oversized, oversizedValidationDecision(item.CorrectionIndex))
continue
}
usable = append(usable, fitted.item)
if simple != nil {
costs = append(costs, fitted.cost)
} else {
sizes = append(sizes, fitted.tokens)
}
}
if simple != nil {
batches, err := chunkSimpleValidationItems(usable, costs, budget, simple)
return batches, oversized, err
}
for i := range items {
items[i] = fitValidationContext(items[i], budget, estimator)
}
oversized := oversizedValidationDecisions(items, budget, estimator)
usable := filterItemsByDecision(items, oversized)
if len(usable) == 0 {
return nil, oversized, nil
}
batches, err := ChunkLLMValidationItems(usable, budget, estimator)
batches, err := chunkSerializedValidationItems(usable, sizes, budget, estimator)
return batches, oversized, err
}
@@ -61,26 +61,40 @@ func ChunkLLMValidationItems(items []LLMValidationItem, maxPromptTokens int, est
return chunkSimpleValidationItems(items, nil, maxPromptTokens, simple)
}
batches := make([]LLMValidationBatch, 0)
current := make([]LLMValidationItem, 0)
return chunkSerializedValidationItems(items, nil, maxPromptTokens, estimator)
}
func chunkSerializedValidationItems(items []LLMValidationItem, sizes []int, maxPromptTokens int, estimator chunking.TokenEstimator) ([]LLMValidationBatch, error) {
var batches []LLMValidationBatch
var current []LLMValidationItem
currentTokens := 0
for _, item := range items {
singleTokens, err := estimateBatchTokens(estimator, []LLMValidationItem{item})
if err != nil {
return nil, err
for i, item := range items {
var singleTokens int
if sizes != nil {
singleTokens = sizes[i]
} else {
var err error
singleTokens, err = estimateBatchTokens(estimator, []LLMValidationItem{item})
if err != nil {
return nil, err
}
}
if singleTokens > maxPromptTokens {
return nil, fmt.Errorf("single validation proposal exceeds max prompt tokens")
}
if len(current) == 0 {
current, currentTokens = []LLMValidationItem{item}, singleTokens
continue
}
candidate := append(append([]LLMValidationItem(nil), current...), item)
candidateTokens, err := estimateBatchTokens(estimator, candidate)
if err != nil {
return nil, err
}
if len(current) > 0 && candidateTokens > maxPromptTokens {
if candidateTokens > maxPromptTokens {
batches = append(batches, LLMValidationBatch{
BatchIndex: len(batches),
Items: append([]LLMValidationItem(nil), current...),

View File

@@ -39,52 +39,70 @@ func measureValidationItem(estimator *chunking.SimpleTokenEstimator, item LLMVal
return measureJSON(estimator, payload), nil
}
func measureContextPrefixes[T any](estimator *chunking.SimpleTokenEstimator, field string, entries []T) ([]validationCost, error) {
prefixes := make([]validationCost, len(entries)+1)
cost := measureJSON(estimator, []byte(`,"`+field+`":[]`))
for i, entry := range entries {
payload, err := json.Marshal(entry)
if err != nil {
return nil, fmt.Errorf("marshal validation %s: %w", field, err)
}
if i > 0 {
cost.punctuation++ // separator between entries
}
cost = cost.add(measureJSON(estimator, payload))
prefixes[i+1] = cost
}
return prefixes, nil
type fittedValidationItem struct {
item LLMValidationItem
cost validationCost
tokens int
}
func fitSimpleValidationContext(item LLMValidationItem, budget int, estimator *chunking.SimpleTokenEstimator) (LLMValidationItem, validationCost, error) {
required := item
required.NeighboringSegments, required.GlossaryEntries = nil, nil
base, err := measureValidationItem(estimator, required)
if err != nil {
return item, validationCost{}, err
// fitValidationContext tries the required payload followed by glossary entries
// and then neighboring segments. The first excess ends the usable prefix.
func fitValidationContext(item LLMValidationItem, budget int, estimator chunking.TokenEstimator) (fittedValidationItem, error) {
fitted := fittedValidationItem{item: item}
fitted.item.GlossaryEntries = item.GlossaryEntries[:0]
fitted.item.NeighboringSegments = item.NeighboringSegments[:0]
simple, _ := estimator.(*chunking.SimpleTokenEstimator)
var err error
if simple != nil {
fitted.cost, err = measureValidationItem(simple, fitted.item)
fitted.tokens = fitted.cost.arrayTokens()
} else {
fitted.tokens, err = estimateBatchTokens(estimator, []LLMValidationItem{fitted.item})
}
if len(item.NeighboringSegments)+len(item.GlossaryEntries) == 0 {
return item, base, nil
if err != nil || fitted.tokens > budget {
return fitted, err
}
neighbors, err := measureContextPrefixes(estimator, "neighboring_segments", item.NeighboringSegments)
if err != nil {
return item, validationCost{}, err
try := func(candidate LLMValidationItem, field string, first bool, entry any) (bool, error) {
cost := fitted.cost
var tokens int
if simple != nil {
payload, err := json.Marshal(entry)
if err != nil {
return false, fmt.Errorf("marshal validation %s: %w", field, err)
}
if first {
cost = cost.add(measureJSON(simple, []byte(`,"`+field+`":[]`)))
} else {
cost.punctuation++ // separator between entries
}
cost = cost.add(measureJSON(simple, payload))
tokens = cost.arrayTokens()
} else {
var err error
tokens, err = estimateBatchTokens(estimator, []LLMValidationItem{candidate})
if err != nil {
return false, err
}
}
if tokens > budget {
return false, nil
}
fitted = fittedValidationItem{item: candidate, cost: cost, tokens: tokens}
return true, nil
}
glossary, err := measureContextPrefixes(estimator, "glossary_entries", item.GlossaryEntries)
if err != nil {
return item, validationCost{}, err
for i, entry := range item.GlossaryEntries {
candidate := fitted.item
candidate.GlossaryEntries = item.GlossaryEntries[:i+1]
if ok, err := try(candidate, "glossary_entries", i == 0, entry); !ok {
return fitted, err
}
}
n, g := len(item.NeighboringSegments), len(item.GlossaryEntries)
cost := base.add(neighbors[n]).add(glossary[g])
for n > 0 && cost.arrayTokens() > budget {
n--
cost = base.add(neighbors[n]).add(glossary[g])
for i, entry := range item.NeighboringSegments {
candidate := fitted.item
candidate.NeighboringSegments = item.NeighboringSegments[:i+1]
if ok, err := try(candidate, "neighboring_segments", i == 0, entry); !ok {
return fitted, err
}
}
for g > 0 && cost.arrayTokens() > budget {
g--
cost = base.add(glossary[g])
}
item.NeighboringSegments = item.NeighboringSegments[:n]
item.GlossaryEntries = item.GlossaryEntries[:g]
return item, cost, nil
return fitted, nil
}

View File

@@ -6,13 +6,70 @@ import (
"slices"
"strings"
"testing"
"unicode"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
)
// Wrapping the same estimator exercises budgeting through exact serialization.
type serializedBudgetEstimator struct{ *chunking.SimpleTokenEstimator }
// Count the serialized payload independently from the production token estimator.
func serializedTokenCount(t *testing.T, items []LLMValidationItem) int {
t.Helper()
local := slices.Clone(items)
for i := range local {
local[i].CorrectionIndex = i + 1
}
payload, err := json.Marshal(local)
if err != nil {
t.Fatal(err)
}
text := string(payload)
punctuation := 0
for _, r := range text {
if unicode.IsPunct(r) && r != '\'' && r != '-' && r != '_' {
punctuation++
}
}
return len(strings.Fields(text)) + punctuation/2
}
// Check the same context priority by trimming from the end of serialized
// payloads. This does not use the production prefix fitter or batch builder.
func serializedBudgetOracle(t *testing.T, items []LLMValidationItem, budget int) ([]LLMValidationBatch, []int) {
t.Helper()
var batches []LLMValidationBatch
var oversized []int
var current []LLMValidationItem
closeBatch := func() {
batches = append(batches, LLMValidationBatch{BatchIndex: len(batches), Items: current, TokenCount: serializedTokenCount(t, current)})
}
for _, item := range items {
for serializedTokenCount(t, []LLMValidationItem{item}) > budget {
if n := len(item.NeighboringSegments); n > 0 {
item.NeighboringSegments = item.NeighboringSegments[:n-1]
} else if n := len(item.GlossaryEntries); n > 0 {
item.GlossaryEntries = item.GlossaryEntries[:n-1]
} else {
break
}
}
if serializedTokenCount(t, []LLMValidationItem{item}) > budget {
oversized = append(oversized, item.CorrectionIndex)
continue
}
candidate := append(slices.Clone(current), item)
if len(current) > 0 && serializedTokenCount(t, candidate) > budget {
closeBatch()
current = []LLMValidationItem{item}
} else {
current = candidate
}
}
if len(current) > 0 {
closeBatch()
}
return batches, oversized
}
func TestComposedValidationBudgetMatchesSerializedPayload(t *testing.T) {
items := make([]LLMValidationItem, 12)
@@ -38,17 +95,20 @@ func TestComposedValidationBudgetMatchesSerializedPayload(t *testing.T) {
items[1].CorrectedSegmentText = strings.Repeat("longer candidate. ", 12)
simple := chunking.NewSimpleTokenEstimator()
serialized := serializedBudgetEstimator{simple}
full, err := estimateBatchTokens(serialized, items)
if err != nil {
t.Fatal(err)
}
full := serializedTokenCount(t, items)
for budget := 1; budget <= full; budget++ {
got, rejected, err := prepareValidationBatches(slices.Clone(items), budget, simple)
want, wantRejected, serializedErr := prepareValidationBatches(slices.Clone(items), budget, serialized)
if err != nil || serializedErr != nil || !reflect.DeepEqual(got, want) || !reflect.DeepEqual(rejected, wantRejected) {
t.Fatalf("budget %d: composed=%+v rejected=%+v err=%v; serialized=%+v rejected=%+v err=%v",
budget, got, rejected, err, want, wantRejected, serializedErr)
got, rejected, err := prepareValidationBatches(items, budget, simple)
want, wantRejected := serializedBudgetOracle(t, items, budget)
var rejectedIndices []int
for _, decision := range rejected {
if decision.Disposition != DispositionUnavailable || decision.ReasonCode != ReasonValidatorInputTooLarge {
t.Fatalf("unexpected over-budget decision: %+v", decision)
}
rejectedIndices = append(rejectedIndices, decision.ProposalIndex)
}
if err != nil || !reflect.DeepEqual(got, want) || !slices.Equal(rejectedIndices, wantRejected) {
t.Fatalf("budget %d: composed=%+v rejected=%+v err=%v; serialized=%+v rejected=%+v",
budget, got, rejected, err, want, wantRejected)
}
}
@@ -73,3 +133,37 @@ func TestComposedValidationBudgetMatchesSerializedPayload(t *testing.T) {
}
}
}
type countingEstimator struct {
chunking.SimpleTokenEstimator
measurements int
}
func (e *countingEstimator) EstimateTokens(text string) int {
e.measurements++
return e.SimpleTokenEstimator.EstimateTokens(text)
}
func TestValidationFitsRequiredDataBeforeScanningOptionalContext(t *testing.T) {
item := LLMValidationItem{CorrectionIndex: 7, OriginalText: "teh", CorrectedText: "the"}
first := glossary.Entry{ID: "keep", Name: "one"}
prefix := item
prefix.GlossaryEntries = []glossary.Entry{first}
budget := serializedTokenCount(t, []LLMValidationItem{prefix})
item.GlossaryEntries = append([]glossary.Entry{first, {Description: strings.Repeat("large body ", 500)}}, make([]glossary.Entry, 5000)...)
item.NeighboringSegments = []NeighboringSegment{{Text: "neighbor"}}
estimator := &countingEstimator{}
fitted, err := fitValidationContext(item, budget, estimator)
if err != nil || len(fitted.item.GlossaryEntries) != 1 || fitted.item.GlossaryEntries[0].ID != first.ID ||
len(fitted.item.NeighboringSegments) != 0 || fitted.tokens != budget {
t.Fatalf("incorrect usable prefix: %+v (%v)", fitted, err)
}
if estimator.measurements != 3 {
t.Fatalf("measured beyond the first excess: %d measurements", estimator.measurements)
}
estimator.measurements = 0
fitted, err = fitValidationContext(item, 1, estimator)
if err != nil || fitted.tokens <= 1 || estimator.measurements != 1 {
t.Fatalf("oversized required data: tokens=%d measurements=%d error=%v", fitted.tokens, estimator.measurements, err)
}
}

View File

@@ -3,7 +3,6 @@ package validators
import (
"slices"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
)
@@ -64,24 +63,6 @@ func (view glossaryContext) validationContext(segments []schema.Segment, positio
return neighbors, entries
}
// fitValidationContext drops optional context before treating required candidate
// text as oversized. Glossary definitions take priority over adjacent segments.
func fitValidationContext(item LLMValidationItem, budget int, estimator chunking.TokenEstimator) LLMValidationItem {
for {
tokens, err := estimateBatchTokens(estimator, []LLMValidationItem{item})
if err != nil || tokens <= budget {
return item
}
if len(item.NeighboringSegments) > 0 {
item.NeighboringSegments = item.NeighboringSegments[:len(item.NeighboringSegments)-1]
} else if len(item.GlossaryEntries) > 0 {
item.GlossaryEntries = item.GlossaryEntries[:len(item.GlossaryEntries)-1]
} else {
return item
}
}
}
func neighboringSegment(position string, segment schema.Segment) NeighboringSegment {
return NeighboringSegment{Position: position, Speaker: segment.Speaker, Text: segment.Text, Categories: segment.Categories}
}

View File

@@ -73,7 +73,11 @@ func TestValidationContextFitsBudgetWithoutDiscardingCandidate(t *testing.T) {
enriched := base
enriched.NeighboringSegments = []NeighboringSegment{{Position: "following", Text: strings.Repeat("long context ", 300)}}
enriched.GlossaryEntries = []glossary.Entry{{Name: "cat", Description: strings.Repeat("long definition ", 300)}}
fitted := fitValidationContext(enriched, required, estimator)
fit, err := fitValidationContext(enriched, required, estimator)
if err != nil {
t.Fatal(err)
}
fitted := fit.item
tokens, err := estimateBatchTokens(estimator, []LLMValidationItem{fitted})
if err != nil || tokens > required || fitted.Speaker != base.Speaker || fitted.OriginalSegmentText != base.OriginalSegmentText || fitted.CorrectedSegmentText != base.CorrectedSegmentText {
t.Fatalf("fitted: %+v tokens=%d err=%v", fitted, tokens, err)

View File

@@ -330,40 +330,10 @@ func reconcileLLMResponse(items []LLMValidationItem, response LLMValidationRespo
return result
}
func oversizedValidationDecisions(items []LLMValidationItem, maxPromptTokens int, estimator chunking.TokenEstimator) []Decision {
var out []Decision
for _, item := range items {
singleTokens, err := estimateBatchTokens(estimator, []LLMValidationItem{item})
if err != nil || singleTokens <= maxPromptTokens {
continue
}
out = append(out, oversizedValidationDecision(item.CorrectionIndex))
}
return out
}
func oversizedValidationDecision(index int) Decision {
return unavailable(index, ReasonValidatorInputTooLarge, "validation input exceeds max prompt tokens")
}
func filterItemsByDecision(items []LLMValidationItem, decisions []Decision) []LLMValidationItem {
if len(decisions) == 0 {
return append([]LLMValidationItem(nil), items...)
}
rejected := make(map[int]struct{}, len(decisions))
for _, decision := range decisions {
rejected[decision.ProposalIndex] = struct{}{}
}
out := make([]LLMValidationItem, 0, len(items))
for _, item := range items {
if _, ok := rejected[item.CorrectionIndex]; ok {
continue
}
out = append(out, item)
}
return out
}
func rejectBatch(items []LLMValidationItem, reasonCode string, message string) []Decision {
out := make([]Decision, 0, len(items))
for _, item := range items {