Measure only the usable validation context prefix

This commit is contained in:
2026-11-16 14:14:21 +00:00
parent 0c144fd5a5
commit b1d1b8302c
8 changed files with 229 additions and 144 deletions

View File

@@ -6,13 +6,70 @@ import (
"slices"
"strings"
"testing"
"unicode"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
)
// Wrapping the same estimator exercises budgeting through exact serialization.
type serializedBudgetEstimator struct{ *chunking.SimpleTokenEstimator }
// Count the serialized payload independently from the production token estimator.
func serializedTokenCount(t *testing.T, items []LLMValidationItem) int {
t.Helper()
local := slices.Clone(items)
for i := range local {
local[i].CorrectionIndex = i + 1
}
payload, err := json.Marshal(local)
if err != nil {
t.Fatal(err)
}
text := string(payload)
punctuation := 0
for _, r := range text {
if unicode.IsPunct(r) && r != '\'' && r != '-' && r != '_' {
punctuation++
}
}
return len(strings.Fields(text)) + punctuation/2
}
// Check the same context priority by trimming from the end of serialized
// payloads. This does not use the production prefix fitter or batch builder.
func serializedBudgetOracle(t *testing.T, items []LLMValidationItem, budget int) ([]LLMValidationBatch, []int) {
t.Helper()
var batches []LLMValidationBatch
var oversized []int
var current []LLMValidationItem
closeBatch := func() {
batches = append(batches, LLMValidationBatch{BatchIndex: len(batches), Items: current, TokenCount: serializedTokenCount(t, current)})
}
for _, item := range items {
for serializedTokenCount(t, []LLMValidationItem{item}) > budget {
if n := len(item.NeighboringSegments); n > 0 {
item.NeighboringSegments = item.NeighboringSegments[:n-1]
} else if n := len(item.GlossaryEntries); n > 0 {
item.GlossaryEntries = item.GlossaryEntries[:n-1]
} else {
break
}
}
if serializedTokenCount(t, []LLMValidationItem{item}) > budget {
oversized = append(oversized, item.CorrectionIndex)
continue
}
candidate := append(slices.Clone(current), item)
if len(current) > 0 && serializedTokenCount(t, candidate) > budget {
closeBatch()
current = []LLMValidationItem{item}
} else {
current = candidate
}
}
if len(current) > 0 {
closeBatch()
}
return batches, oversized
}
func TestComposedValidationBudgetMatchesSerializedPayload(t *testing.T) {
items := make([]LLMValidationItem, 12)
@@ -38,17 +95,20 @@ func TestComposedValidationBudgetMatchesSerializedPayload(t *testing.T) {
items[1].CorrectedSegmentText = strings.Repeat("longer candidate. ", 12)
simple := chunking.NewSimpleTokenEstimator()
serialized := serializedBudgetEstimator{simple}
full, err := estimateBatchTokens(serialized, items)
if err != nil {
t.Fatal(err)
}
full := serializedTokenCount(t, items)
for budget := 1; budget <= full; budget++ {
got, rejected, err := prepareValidationBatches(slices.Clone(items), budget, simple)
want, wantRejected, serializedErr := prepareValidationBatches(slices.Clone(items), budget, serialized)
if err != nil || serializedErr != nil || !reflect.DeepEqual(got, want) || !reflect.DeepEqual(rejected, wantRejected) {
t.Fatalf("budget %d: composed=%+v rejected=%+v err=%v; serialized=%+v rejected=%+v err=%v",
budget, got, rejected, err, want, wantRejected, serializedErr)
got, rejected, err := prepareValidationBatches(items, budget, simple)
want, wantRejected := serializedBudgetOracle(t, items, budget)
var rejectedIndices []int
for _, decision := range rejected {
if decision.Disposition != DispositionUnavailable || decision.ReasonCode != ReasonValidatorInputTooLarge {
t.Fatalf("unexpected over-budget decision: %+v", decision)
}
rejectedIndices = append(rejectedIndices, decision.ProposalIndex)
}
if err != nil || !reflect.DeepEqual(got, want) || !slices.Equal(rejectedIndices, wantRejected) {
t.Fatalf("budget %d: composed=%+v rejected=%+v err=%v; serialized=%+v rejected=%+v",
budget, got, rejected, err, want, wantRejected)
}
}
@@ -73,3 +133,37 @@ func TestComposedValidationBudgetMatchesSerializedPayload(t *testing.T) {
}
}
}
type countingEstimator struct {
chunking.SimpleTokenEstimator
measurements int
}
func (e *countingEstimator) EstimateTokens(text string) int {
e.measurements++
return e.SimpleTokenEstimator.EstimateTokens(text)
}
func TestValidationFitsRequiredDataBeforeScanningOptionalContext(t *testing.T) {
item := LLMValidationItem{CorrectionIndex: 7, OriginalText: "teh", CorrectedText: "the"}
first := glossary.Entry{ID: "keep", Name: "one"}
prefix := item
prefix.GlossaryEntries = []glossary.Entry{first}
budget := serializedTokenCount(t, []LLMValidationItem{prefix})
item.GlossaryEntries = append([]glossary.Entry{first, {Description: strings.Repeat("large body ", 500)}}, make([]glossary.Entry, 5000)...)
item.NeighboringSegments = []NeighboringSegment{{Text: "neighbor"}}
estimator := &countingEstimator{}
fitted, err := fitValidationContext(item, budget, estimator)
if err != nil || len(fitted.item.GlossaryEntries) != 1 || fitted.item.GlossaryEntries[0].ID != first.ID ||
len(fitted.item.NeighboringSegments) != 0 || fitted.tokens != budget {
t.Fatalf("incorrect usable prefix: %+v (%v)", fitted, err)
}
if estimator.measurements != 3 {
t.Fatalf("measured beyond the first excess: %d measurements", estimator.measurements)
}
estimator.measurements = 0
fitted, err = fitValidationContext(item, 1, estimator)
if err != nil || fitted.tokens <= 1 || estimator.measurements != 1 {
t.Fatalf("oversized required data: tokens=%d measurements=%d error=%v", fitted.tokens, estimator.measurements, err)
}
}