Files
audita/internal/framework/validators/llm_budget_test.go

196 lines
7.2 KiB
Go
Raw Normal View History

package validators
import (
"bytes"
"encoding/json"
"slices"
"strings"
"testing"
"unicode"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
)
// Count the serialized payload independently from the production token estimator.
func serializedTokenCount(t *testing.T, items []LLMValidationItem) int {
t.Helper()
local := slices.Clone(items)
for i := range local {
local[i].CorrectionIndex = i + 1
}
payload, err := json.Marshal(local)
if err != nil {
t.Fatal(err)
}
text := string(payload)
punctuation := 0
for _, r := range text {
if unicode.IsPunct(r) && r != '\'' && r != '-' && r != '_' {
punctuation++
}
}
return len(strings.Fields(text)) + punctuation/2
}
// Check the same context priority by trimming from the end of serialized
// payloads. This does not use the production prefix fitter or batch builder.
func serializedBudgetOracle(t *testing.T, items []LLMValidationItem, budget int) ([]LLMValidationBatch, []int) {
t.Helper()
var batches []LLMValidationBatch
var oversized []int
var current []LLMValidationItem
closeBatch := func() {
batches = append(batches, LLMValidationBatch{BatchIndex: len(batches), Items: current, TokenCount: serializedTokenCount(t, current)})
}
for _, item := range items {
for serializedTokenCount(t, []LLMValidationItem{item}) > budget {
if n := len(item.NeighboringSegments); n > 0 {
item.NeighboringSegments = item.NeighboringSegments[:n-1]
} else if n := len(item.GlossaryEntries); n > 0 {
item.GlossaryEntries = item.GlossaryEntries[:n-1]
} else {
break
}
}
if serializedTokenCount(t, []LLMValidationItem{item}) > budget {
oversized = append(oversized, item.CorrectionIndex)
continue
}
candidate := append(slices.Clone(current), item)
if len(current) > 0 && serializedTokenCount(t, candidate) > budget {
closeBatch()
current = []LLMValidationItem{item}
} else {
current = candidate
}
}
if len(current) > 0 {
closeBatch()
}
return batches, oversized
}
func TestComposedValidationBudgetMatchesSerializedPayload(t *testing.T) {
items := make([]LLMValidationItem, 12)
for i := range items {
items[i] = LLMValidationItem{
CorrectionIndex: 100 + i*7,
Speaker: "Guest \"A\" — session host",
OriginalText: "one\u00a0two!",
CorrectedText: "one two",
OriginalSegmentText: "she said \"one\u00a0two!\"\n",
CorrectedSegmentText: "she said <one & two> — there.",
Categories: []string{"say, then write", "repeat"},
}
}
items[0].NeighboringSegments = []NeighboringSegment{
{Position: "preceding", Text: "spaces\tand escapes\\", Categories: []string{"speaker line"}},
{Position: "following", Text: "‘quoted’\u2028words"},
}
items[0].GlossaryEntries = []glossary.Entry{
{ID: "name", Name: "Name?", Aliases: []string{"N. M.", "Name's"}},
{ID: "two", Name: "One Two", Description: "some\u2003meaning…"},
}
items[1].CorrectedSegmentText = strings.Repeat("longer candidate. ", 12)
simple := chunking.NewSimpleTokenEstimator()
full := serializedTokenCount(t, items)
for budget := 1; budget <= full; budget++ {
got, rejected, err := prepareValidationBatches(items, budget, simple, nil)
want, wantRejected := serializedBudgetOracle(t, items, budget)
var rejectedIndices []int
for _, decision := range rejected {
if decision.Disposition != DispositionUnavailable || decision.ReasonCode != ReasonValidatorInputTooLarge {
t.Fatalf("unexpected over-budget decision: %+v", decision)
}
rejectedIndices = append(rejectedIndices, decision.ProposalIndex)
}
gotJSON, gotErr := json.Marshal(got)
wantJSON, wantErr := json.Marshal(want)
if err != nil || gotErr != nil || wantErr != nil || !bytes.Equal(gotJSON, wantJSON) || !slices.Equal(rejectedIndices, wantRejected) {
t.Fatalf("budget %d: composed=%+v rejected=%+v err=%v; serialized=%+v rejected=%+v",
budget, got, rejected, err, want, wantRejected)
}
}
batches, err := ChunkLLMValidationItems(items, full, simple)
if err != nil || len(batches) != 1 || batches[0].TokenCount != full {
t.Fatalf("full batch: %+v, %v", batches, err)
}
variables, err := BuildValidationVariables(batches[0].Items, "")
if err != nil {
t.Fatal(err)
}
var payload []LLMValidationItem
if err := json.Unmarshal([]byte(variables["PayloadJSON"]), &payload); err != nil {
t.Fatal(err)
}
if len(payload) != len(items) {
t.Fatalf("sent %d candidates, want %d", len(payload), len(items))
}
for i := range payload {
if payload[i].CorrectionIndex != i+1 || batches[0].Items[i].CorrectionIndex != items[i].CorrectionIndex {
t.Fatalf("candidate %d lost its local or run reference", i)
}
}
}
type countingEstimator struct {
chunking.SimpleTokenEstimator
measurements int
}
func (e *countingEstimator) EstimateTokens(text string) int {
e.measurements++
return e.SimpleTokenEstimator.EstimateTokens(text)
}
func TestValidationFitsRequiredDataBeforeScanningOptionalContext(t *testing.T) {
item := LLMValidationItem{CorrectionIndex: 7, OriginalText: "teh", CorrectedText: "the"}
first := glossary.Entry{ID: "keep", Name: "one"}
prefix := item
prefix.GlossaryEntries = []glossary.Entry{first}
budget := serializedTokenCount(t, []LLMValidationItem{prefix})
item.GlossaryEntries = append([]glossary.Entry{first, {Description: strings.Repeat("large body ", 500)}}, make([]glossary.Entry, 5000)...)
item.NeighboringSegments = []NeighboringSegment{{Text: "neighbor"}}
estimator := &countingEstimator{}
fitted, err := fitValidationContext(item, budget, estimator, nil)
if err != nil || len(fitted.item.GlossaryEntries) != 1 || fitted.item.GlossaryEntries[0].ID != first.ID ||
len(fitted.item.NeighboringSegments) != 0 || fitted.tokens != budget {
t.Fatalf("incorrect usable prefix: %+v (%v)", fitted, err)
}
if estimator.measurements != 3 {
t.Fatalf("measured beyond the first excess: %d measurements", estimator.measurements)
}
estimator.measurements = 0
fitted, err = fitValidationContext(item, 1, estimator, nil)
if err != nil || fitted.tokens <= 1 || estimator.measurements != 1 {
t.Fatalf("oversized required data: tokens=%d measurements=%d error=%v", fitted.tokens, estimator.measurements, err)
}
}
func TestValidationSkipsOptionalContextWithoutBudget(t *testing.T) {
item := LLMValidationItem{CorrectionIndex: 7, OriginalText: "teh", CorrectedText: "the"}
estimator := chunking.NewSimpleTokenEstimator()
required := serializedTokenCount(t, []LLMValidationItem{item})
loads := 0
optional := func(LLMValidationItem) ([]NeighboringSegment, []glossary.Entry) {
loads++
return nil, []glossary.Entry{{Name: "optional"}}
}
for _, budget := range []int{required - 1, required} {
batches, oversized, err := prepareValidationBatches([]LLMValidationItem{item}, budget, estimator, optional)
if err != nil || loads != 0 {
t.Fatalf("budget=%d loads=%d error=%v", budget, loads, err)
}
if budget < required {
if len(batches) != 0 || len(oversized) != 1 {
t.Fatalf("oversized result: batches=%+v rejected=%+v", batches, oversized)
}
} else if len(batches) != 1 || len(oversized) != 0 || batches[0].TokenCount != required {
t.Fatalf("exact fit: batches=%+v rejected=%+v", batches, oversized)
}
}
}