Measure only the usable validation context prefix
This commit is contained in:
@@ -6,13 +6,70 @@ import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"unicode"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
|
||||
)
|
||||
|
||||
// Wrapping the same estimator exercises budgeting through exact serialization.
|
||||
type serializedBudgetEstimator struct{ *chunking.SimpleTokenEstimator }
|
||||
// Count the serialized payload independently from the production token estimator.
|
||||
func serializedTokenCount(t *testing.T, items []LLMValidationItem) int {
|
||||
t.Helper()
|
||||
local := slices.Clone(items)
|
||||
for i := range local {
|
||||
local[i].CorrectionIndex = i + 1
|
||||
}
|
||||
payload, err := json.Marshal(local)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
text := string(payload)
|
||||
punctuation := 0
|
||||
for _, r := range text {
|
||||
if unicode.IsPunct(r) && r != '\'' && r != '-' && r != '_' {
|
||||
punctuation++
|
||||
}
|
||||
}
|
||||
return len(strings.Fields(text)) + punctuation/2
|
||||
}
|
||||
|
||||
// Check the same context priority by trimming from the end of serialized
|
||||
// payloads. This does not use the production prefix fitter or batch builder.
|
||||
func serializedBudgetOracle(t *testing.T, items []LLMValidationItem, budget int) ([]LLMValidationBatch, []int) {
|
||||
t.Helper()
|
||||
var batches []LLMValidationBatch
|
||||
var oversized []int
|
||||
var current []LLMValidationItem
|
||||
closeBatch := func() {
|
||||
batches = append(batches, LLMValidationBatch{BatchIndex: len(batches), Items: current, TokenCount: serializedTokenCount(t, current)})
|
||||
}
|
||||
for _, item := range items {
|
||||
for serializedTokenCount(t, []LLMValidationItem{item}) > budget {
|
||||
if n := len(item.NeighboringSegments); n > 0 {
|
||||
item.NeighboringSegments = item.NeighboringSegments[:n-1]
|
||||
} else if n := len(item.GlossaryEntries); n > 0 {
|
||||
item.GlossaryEntries = item.GlossaryEntries[:n-1]
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
if serializedTokenCount(t, []LLMValidationItem{item}) > budget {
|
||||
oversized = append(oversized, item.CorrectionIndex)
|
||||
continue
|
||||
}
|
||||
candidate := append(slices.Clone(current), item)
|
||||
if len(current) > 0 && serializedTokenCount(t, candidate) > budget {
|
||||
closeBatch()
|
||||
current = []LLMValidationItem{item}
|
||||
} else {
|
||||
current = candidate
|
||||
}
|
||||
}
|
||||
if len(current) > 0 {
|
||||
closeBatch()
|
||||
}
|
||||
return batches, oversized
|
||||
}
|
||||
|
||||
func TestComposedValidationBudgetMatchesSerializedPayload(t *testing.T) {
|
||||
items := make([]LLMValidationItem, 12)
|
||||
@@ -38,17 +95,20 @@ func TestComposedValidationBudgetMatchesSerializedPayload(t *testing.T) {
|
||||
items[1].CorrectedSegmentText = strings.Repeat("longer candidate. ", 12)
|
||||
|
||||
simple := chunking.NewSimpleTokenEstimator()
|
||||
serialized := serializedBudgetEstimator{simple}
|
||||
full, err := estimateBatchTokens(serialized, items)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
full := serializedTokenCount(t, items)
|
||||
for budget := 1; budget <= full; budget++ {
|
||||
got, rejected, err := prepareValidationBatches(slices.Clone(items), budget, simple)
|
||||
want, wantRejected, serializedErr := prepareValidationBatches(slices.Clone(items), budget, serialized)
|
||||
if err != nil || serializedErr != nil || !reflect.DeepEqual(got, want) || !reflect.DeepEqual(rejected, wantRejected) {
|
||||
t.Fatalf("budget %d: composed=%+v rejected=%+v err=%v; serialized=%+v rejected=%+v err=%v",
|
||||
budget, got, rejected, err, want, wantRejected, serializedErr)
|
||||
got, rejected, err := prepareValidationBatches(items, budget, simple)
|
||||
want, wantRejected := serializedBudgetOracle(t, items, budget)
|
||||
var rejectedIndices []int
|
||||
for _, decision := range rejected {
|
||||
if decision.Disposition != DispositionUnavailable || decision.ReasonCode != ReasonValidatorInputTooLarge {
|
||||
t.Fatalf("unexpected over-budget decision: %+v", decision)
|
||||
}
|
||||
rejectedIndices = append(rejectedIndices, decision.ProposalIndex)
|
||||
}
|
||||
if err != nil || !reflect.DeepEqual(got, want) || !slices.Equal(rejectedIndices, wantRejected) {
|
||||
t.Fatalf("budget %d: composed=%+v rejected=%+v err=%v; serialized=%+v rejected=%+v",
|
||||
budget, got, rejected, err, want, wantRejected)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -73,3 +133,37 @@ func TestComposedValidationBudgetMatchesSerializedPayload(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type countingEstimator struct {
|
||||
chunking.SimpleTokenEstimator
|
||||
measurements int
|
||||
}
|
||||
|
||||
func (e *countingEstimator) EstimateTokens(text string) int {
|
||||
e.measurements++
|
||||
return e.SimpleTokenEstimator.EstimateTokens(text)
|
||||
}
|
||||
|
||||
func TestValidationFitsRequiredDataBeforeScanningOptionalContext(t *testing.T) {
|
||||
item := LLMValidationItem{CorrectionIndex: 7, OriginalText: "teh", CorrectedText: "the"}
|
||||
first := glossary.Entry{ID: "keep", Name: "one"}
|
||||
prefix := item
|
||||
prefix.GlossaryEntries = []glossary.Entry{first}
|
||||
budget := serializedTokenCount(t, []LLMValidationItem{prefix})
|
||||
item.GlossaryEntries = append([]glossary.Entry{first, {Description: strings.Repeat("large body ", 500)}}, make([]glossary.Entry, 5000)...)
|
||||
item.NeighboringSegments = []NeighboringSegment{{Text: "neighbor"}}
|
||||
estimator := &countingEstimator{}
|
||||
fitted, err := fitValidationContext(item, budget, estimator)
|
||||
if err != nil || len(fitted.item.GlossaryEntries) != 1 || fitted.item.GlossaryEntries[0].ID != first.ID ||
|
||||
len(fitted.item.NeighboringSegments) != 0 || fitted.tokens != budget {
|
||||
t.Fatalf("incorrect usable prefix: %+v (%v)", fitted, err)
|
||||
}
|
||||
if estimator.measurements != 3 {
|
||||
t.Fatalf("measured beyond the first excess: %d measurements", estimator.measurements)
|
||||
}
|
||||
estimator.measurements = 0
|
||||
fitted, err = fitValidationContext(item, 1, estimator)
|
||||
if err != nil || fitted.tokens <= 1 || estimator.measurements != 1 {
|
||||
t.Fatalf("oversized required data: tokens=%d measurements=%d error=%v", fitted.tokens, estimator.measurements, err)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user