Files
audita/internal/framework/validators/llm_budget_test.go
Eric Rakestraw da5b8913d1
All checks were successful
ci/woodpecker/tag/release Pipeline was successful
Migrate Audita to the v4 module path
2026-11-17 19:57:22 +00:00

196 lines
7.2 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package validators
import (
"bytes"
"encoding/json"
"slices"
"strings"
"testing"
"unicode"
"gitea.maximumdirect.net/eric/audita/v4/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/v4/internal/core/glossary"
)
// Count the serialized payload independently from the production token estimator.
func serializedTokenCount(t *testing.T, items []LLMValidationItem) int {
t.Helper()
local := slices.Clone(items)
for i := range local {
local[i].CorrectionIndex = i + 1
}
payload, err := json.Marshal(local)
if err != nil {
t.Fatal(err)
}
text := string(payload)
punctuation := 0
for _, r := range text {
if unicode.IsPunct(r) && r != '\'' && r != '-' && r != '_' {
punctuation++
}
}
return len(strings.Fields(text)) + punctuation/2
}
// Check the same context priority by trimming from the end of serialized
// payloads. This does not use the production prefix fitter or batch builder.
func serializedBudgetOracle(t *testing.T, items []LLMValidationItem, budget int) ([]LLMValidationBatch, []int) {
t.Helper()
var batches []LLMValidationBatch
var oversized []int
var current []LLMValidationItem
closeBatch := func() {
batches = append(batches, LLMValidationBatch{BatchIndex: len(batches), Items: current, TokenCount: serializedTokenCount(t, current)})
}
for _, item := range items {
for serializedTokenCount(t, []LLMValidationItem{item}) > budget {
if n := len(item.NeighboringSegments); n > 0 {
item.NeighboringSegments = item.NeighboringSegments[:n-1]
} else if n := len(item.GlossaryEntries); n > 0 {
item.GlossaryEntries = item.GlossaryEntries[:n-1]
} else {
break
}
}
if serializedTokenCount(t, []LLMValidationItem{item}) > budget {
oversized = append(oversized, item.CorrectionIndex)
continue
}
candidate := append(slices.Clone(current), item)
if len(current) > 0 && serializedTokenCount(t, candidate) > budget {
closeBatch()
current = []LLMValidationItem{item}
} else {
current = candidate
}
}
if len(current) > 0 {
closeBatch()
}
return batches, oversized
}
func TestComposedValidationBudgetMatchesSerializedPayload(t *testing.T) {
items := make([]LLMValidationItem, 12)
for i := range items {
items[i] = LLMValidationItem{
CorrectionIndex: 100 + i*7,
Speaker: "Guest \"A\" — session host",
OriginalText: "one\u00a0two!",
CorrectedText: "one two",
OriginalSegmentText: "she said \"one\u00a0two!\"\n",
CorrectedSegmentText: "she said <one & two> — there.",
Categories: []string{"say, then write", "repeat"},
}
}
items[0].NeighboringSegments = []NeighboringSegment{
{Position: "preceding", Text: "spaces\tand escapes\\", Categories: []string{"speaker line"}},
{Position: "following", Text: "‘quoted’\u2028words"},
}
items[0].GlossaryEntries = []glossary.Entry{
{ID: "name", Name: "Name?", Aliases: []string{"N. M.", "Name's"}},
{ID: "two", Name: "One Two", Description: "some\u2003meaning…"},
}
items[1].CorrectedSegmentText = strings.Repeat("longer candidate. ", 12)
simple := chunking.NewSimpleTokenEstimator()
full := serializedTokenCount(t, items)
for budget := 1; budget <= full; budget++ {
got, rejected, err := prepareValidationBatches(items, budget, simple, nil)
want, wantRejected := serializedBudgetOracle(t, items, budget)
var rejectedIndices []int
for _, decision := range rejected {
if decision.Disposition != DispositionUnavailable || decision.ReasonCode != ReasonValidatorInputTooLarge {
t.Fatalf("unexpected over-budget decision: %+v", decision)
}
rejectedIndices = append(rejectedIndices, decision.ProposalIndex)
}
gotJSON, gotErr := json.Marshal(got)
wantJSON, wantErr := json.Marshal(want)
if err != nil || gotErr != nil || wantErr != nil || !bytes.Equal(gotJSON, wantJSON) || !slices.Equal(rejectedIndices, wantRejected) {
t.Fatalf("budget %d: composed=%+v rejected=%+v err=%v; serialized=%+v rejected=%+v",
budget, got, rejected, err, want, wantRejected)
}
}
batches, err := ChunkLLMValidationItems(items, full, simple)
if err != nil || len(batches) != 1 || batches[0].TokenCount != full {
t.Fatalf("full batch: %+v, %v", batches, err)
}
variables, err := BuildValidationVariables(batches[0].Items, "")
if err != nil {
t.Fatal(err)
}
var payload []LLMValidationItem
if err := json.Unmarshal([]byte(variables["PayloadJSON"]), &payload); err != nil {
t.Fatal(err)
}
if len(payload) != len(items) {
t.Fatalf("sent %d candidates, want %d", len(payload), len(items))
}
for i := range payload {
if payload[i].CorrectionIndex != i+1 || batches[0].Items[i].CorrectionIndex != items[i].CorrectionIndex {
t.Fatalf("candidate %d lost its local or run reference", i)
}
}
}
type countingEstimator struct {
chunking.SimpleTokenEstimator
measurements int
}
func (e *countingEstimator) EstimateTokens(text string) int {
e.measurements++
return e.SimpleTokenEstimator.EstimateTokens(text)
}
func TestValidationFitsRequiredDataBeforeScanningOptionalContext(t *testing.T) {
item := LLMValidationItem{CorrectionIndex: 7, OriginalText: "teh", CorrectedText: "the"}
first := glossary.Entry{ID: "keep", Name: "one"}
prefix := item
prefix.GlossaryEntries = []glossary.Entry{first}
budget := serializedTokenCount(t, []LLMValidationItem{prefix})
item.GlossaryEntries = append([]glossary.Entry{first, {Description: strings.Repeat("large body ", 500)}}, make([]glossary.Entry, 5000)...)
item.NeighboringSegments = []NeighboringSegment{{Text: "neighbor"}}
estimator := &countingEstimator{}
fitted, err := fitValidationContext(item, budget, estimator, nil)
if err != nil || len(fitted.item.GlossaryEntries) != 1 || fitted.item.GlossaryEntries[0].ID != first.ID ||
len(fitted.item.NeighboringSegments) != 0 || fitted.tokens != budget {
t.Fatalf("incorrect usable prefix: %+v (%v)", fitted, err)
}
if estimator.measurements != 3 {
t.Fatalf("measured beyond the first excess: %d measurements", estimator.measurements)
}
estimator.measurements = 0
fitted, err = fitValidationContext(item, 1, estimator, nil)
if err != nil || fitted.tokens <= 1 || estimator.measurements != 1 {
t.Fatalf("oversized required data: tokens=%d measurements=%d error=%v", fitted.tokens, estimator.measurements, err)
}
}
func TestValidationSkipsOptionalContextWithoutBudget(t *testing.T) {
item := LLMValidationItem{CorrectionIndex: 7, OriginalText: "teh", CorrectedText: "the"}
estimator := chunking.NewSimpleTokenEstimator()
required := serializedTokenCount(t, []LLMValidationItem{item})
loads := 0
optional := func(LLMValidationItem) ([]NeighboringSegment, []glossary.Entry) {
loads++
return nil, []glossary.Entry{{Name: "optional"}}
}
for _, budget := range []int{required - 1, required} {
batches, oversized, err := prepareValidationBatches([]LLMValidationItem{item}, budget, estimator, optional)
if err != nil || loads != 0 {
t.Fatalf("budget=%d loads=%d error=%v", budget, loads, err)
}
if budget < required {
if len(batches) != 0 || len(oversized) != 1 {
t.Fatalf("oversized result: batches=%+v rejected=%+v", batches, oversized)
}
} else if len(batches) != 1 || len(oversized) != 0 || batches[0].TokenCount != required {
t.Fatalf("exact fit: batches=%+v rejected=%+v", batches, oversized)
}
}
}