2026-11-10 20:45:38 +00:00
|
|
|
|
package validators
|
|
|
|
|
|
|
|
|
|
|
|
import (
|
2026-11-17 04:25:35 +00:00
|
|
|
|
"bytes"
|
2026-11-10 20:45:38 +00:00
|
|
|
|
"encoding/json"
|
|
|
|
|
|
"slices"
|
|
|
|
|
|
"strings"
|
|
|
|
|
|
"testing"
|
2026-11-16 14:14:21 +00:00
|
|
|
|
"unicode"
|
2026-11-10 20:45:38 +00:00
|
|
|
|
|
|
|
|
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
|
|
|
|
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
|
|
|
|
|
|
)
|
|
|
|
|
|
|
2026-11-16 14:14:21 +00:00
|
|
|
|
// Count the serialized payload independently from the production token estimator.
|
|
|
|
|
|
func serializedTokenCount(t *testing.T, items []LLMValidationItem) int {
|
|
|
|
|
|
t.Helper()
|
|
|
|
|
|
local := slices.Clone(items)
|
|
|
|
|
|
for i := range local {
|
|
|
|
|
|
local[i].CorrectionIndex = i + 1
|
|
|
|
|
|
}
|
|
|
|
|
|
payload, err := json.Marshal(local)
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
|
}
|
|
|
|
|
|
text := string(payload)
|
|
|
|
|
|
punctuation := 0
|
|
|
|
|
|
for _, r := range text {
|
|
|
|
|
|
if unicode.IsPunct(r) && r != '\'' && r != '-' && r != '_' {
|
|
|
|
|
|
punctuation++
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
return len(strings.Fields(text)) + punctuation/2
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// Check the same context priority by trimming from the end of serialized
|
|
|
|
|
|
// payloads. This does not use the production prefix fitter or batch builder.
|
|
|
|
|
|
func serializedBudgetOracle(t *testing.T, items []LLMValidationItem, budget int) ([]LLMValidationBatch, []int) {
|
|
|
|
|
|
t.Helper()
|
|
|
|
|
|
var batches []LLMValidationBatch
|
|
|
|
|
|
var oversized []int
|
|
|
|
|
|
var current []LLMValidationItem
|
|
|
|
|
|
closeBatch := func() {
|
|
|
|
|
|
batches = append(batches, LLMValidationBatch{BatchIndex: len(batches), Items: current, TokenCount: serializedTokenCount(t, current)})
|
|
|
|
|
|
}
|
|
|
|
|
|
for _, item := range items {
|
|
|
|
|
|
for serializedTokenCount(t, []LLMValidationItem{item}) > budget {
|
|
|
|
|
|
if n := len(item.NeighboringSegments); n > 0 {
|
|
|
|
|
|
item.NeighboringSegments = item.NeighboringSegments[:n-1]
|
|
|
|
|
|
} else if n := len(item.GlossaryEntries); n > 0 {
|
|
|
|
|
|
item.GlossaryEntries = item.GlossaryEntries[:n-1]
|
|
|
|
|
|
} else {
|
|
|
|
|
|
break
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
if serializedTokenCount(t, []LLMValidationItem{item}) > budget {
|
|
|
|
|
|
oversized = append(oversized, item.CorrectionIndex)
|
|
|
|
|
|
continue
|
|
|
|
|
|
}
|
|
|
|
|
|
candidate := append(slices.Clone(current), item)
|
|
|
|
|
|
if len(current) > 0 && serializedTokenCount(t, candidate) > budget {
|
|
|
|
|
|
closeBatch()
|
|
|
|
|
|
current = []LLMValidationItem{item}
|
|
|
|
|
|
} else {
|
|
|
|
|
|
current = candidate
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
if len(current) > 0 {
|
|
|
|
|
|
closeBatch()
|
|
|
|
|
|
}
|
|
|
|
|
|
return batches, oversized
|
|
|
|
|
|
}
|
2026-11-10 20:45:38 +00:00
|
|
|
|
|
|
|
|
|
|
func TestComposedValidationBudgetMatchesSerializedPayload(t *testing.T) {
|
|
|
|
|
|
items := make([]LLMValidationItem, 12)
|
|
|
|
|
|
for i := range items {
|
|
|
|
|
|
items[i] = LLMValidationItem{
|
|
|
|
|
|
CorrectionIndex: 100 + i*7,
|
2026-11-11 17:42:30 +00:00
|
|
|
|
Speaker: "Guest \"A\" — session host",
|
2026-11-10 20:45:38 +00:00
|
|
|
|
OriginalText: "one\u00a0two!",
|
|
|
|
|
|
CorrectedText: "one two",
|
|
|
|
|
|
OriginalSegmentText: "she said \"one\u00a0two!\"\n",
|
|
|
|
|
|
CorrectedSegmentText: "she said <one & two> — there.",
|
|
|
|
|
|
Categories: []string{"say, then write", "repeat"},
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
items[0].NeighboringSegments = []NeighboringSegment{
|
|
|
|
|
|
{Position: "preceding", Text: "spaces\tand escapes\\", Categories: []string{"speaker line"}},
|
|
|
|
|
|
{Position: "following", Text: "‘quoted’\u2028words"},
|
|
|
|
|
|
}
|
|
|
|
|
|
items[0].GlossaryEntries = []glossary.Entry{
|
|
|
|
|
|
{ID: "name", Name: "Name?", Aliases: []string{"N. M.", "Name's"}},
|
|
|
|
|
|
{ID: "two", Name: "One Two", Description: "some\u2003meaning…"},
|
|
|
|
|
|
}
|
|
|
|
|
|
items[1].CorrectedSegmentText = strings.Repeat("longer candidate. ", 12)
|
|
|
|
|
|
|
|
|
|
|
|
simple := chunking.NewSimpleTokenEstimator()
|
2026-11-16 14:14:21 +00:00
|
|
|
|
full := serializedTokenCount(t, items)
|
2026-11-10 20:45:38 +00:00
|
|
|
|
for budget := 1; budget <= full; budget++ {
|
2026-11-17 04:25:35 +00:00
|
|
|
|
got, rejected, err := prepareValidationBatches(items, budget, simple, nil)
|
2026-11-16 14:14:21 +00:00
|
|
|
|
want, wantRejected := serializedBudgetOracle(t, items, budget)
|
|
|
|
|
|
var rejectedIndices []int
|
|
|
|
|
|
for _, decision := range rejected {
|
|
|
|
|
|
if decision.Disposition != DispositionUnavailable || decision.ReasonCode != ReasonValidatorInputTooLarge {
|
|
|
|
|
|
t.Fatalf("unexpected over-budget decision: %+v", decision)
|
|
|
|
|
|
}
|
|
|
|
|
|
rejectedIndices = append(rejectedIndices, decision.ProposalIndex)
|
|
|
|
|
|
}
|
2026-11-17 04:25:35 +00:00
|
|
|
|
gotJSON, gotErr := json.Marshal(got)
|
|
|
|
|
|
wantJSON, wantErr := json.Marshal(want)
|
|
|
|
|
|
if err != nil || gotErr != nil || wantErr != nil || !bytes.Equal(gotJSON, wantJSON) || !slices.Equal(rejectedIndices, wantRejected) {
|
2026-11-16 14:14:21 +00:00
|
|
|
|
t.Fatalf("budget %d: composed=%+v rejected=%+v err=%v; serialized=%+v rejected=%+v",
|
|
|
|
|
|
budget, got, rejected, err, want, wantRejected)
|
2026-11-10 20:45:38 +00:00
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
batches, err := ChunkLLMValidationItems(items, full, simple)
|
|
|
|
|
|
if err != nil || len(batches) != 1 || batches[0].TokenCount != full {
|
|
|
|
|
|
t.Fatalf("full batch: %+v, %v", batches, err)
|
|
|
|
|
|
}
|
|
|
|
|
|
variables, err := BuildValidationVariables(batches[0].Items, "")
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
|
}
|
|
|
|
|
|
var payload []LLMValidationItem
|
|
|
|
|
|
if err := json.Unmarshal([]byte(variables["PayloadJSON"]), &payload); err != nil {
|
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
|
}
|
|
|
|
|
|
if len(payload) != len(items) {
|
|
|
|
|
|
t.Fatalf("sent %d candidates, want %d", len(payload), len(items))
|
|
|
|
|
|
}
|
|
|
|
|
|
for i := range payload {
|
|
|
|
|
|
if payload[i].CorrectionIndex != i+1 || batches[0].Items[i].CorrectionIndex != items[i].CorrectionIndex {
|
|
|
|
|
|
t.Fatalf("candidate %d lost its local or run reference", i)
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
2026-11-16 14:14:21 +00:00
|
|
|
|
|
|
|
|
|
|
type countingEstimator struct {
|
|
|
|
|
|
chunking.SimpleTokenEstimator
|
|
|
|
|
|
measurements int
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func (e *countingEstimator) EstimateTokens(text string) int {
|
|
|
|
|
|
e.measurements++
|
|
|
|
|
|
return e.SimpleTokenEstimator.EstimateTokens(text)
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func TestValidationFitsRequiredDataBeforeScanningOptionalContext(t *testing.T) {
|
|
|
|
|
|
item := LLMValidationItem{CorrectionIndex: 7, OriginalText: "teh", CorrectedText: "the"}
|
|
|
|
|
|
first := glossary.Entry{ID: "keep", Name: "one"}
|
|
|
|
|
|
prefix := item
|
|
|
|
|
|
prefix.GlossaryEntries = []glossary.Entry{first}
|
|
|
|
|
|
budget := serializedTokenCount(t, []LLMValidationItem{prefix})
|
|
|
|
|
|
item.GlossaryEntries = append([]glossary.Entry{first, {Description: strings.Repeat("large body ", 500)}}, make([]glossary.Entry, 5000)...)
|
|
|
|
|
|
item.NeighboringSegments = []NeighboringSegment{{Text: "neighbor"}}
|
|
|
|
|
|
estimator := &countingEstimator{}
|
2026-11-17 04:25:35 +00:00
|
|
|
|
fitted, err := fitValidationContext(item, budget, estimator, nil)
|
2026-11-16 14:14:21 +00:00
|
|
|
|
if err != nil || len(fitted.item.GlossaryEntries) != 1 || fitted.item.GlossaryEntries[0].ID != first.ID ||
|
|
|
|
|
|
len(fitted.item.NeighboringSegments) != 0 || fitted.tokens != budget {
|
|
|
|
|
|
t.Fatalf("incorrect usable prefix: %+v (%v)", fitted, err)
|
|
|
|
|
|
}
|
|
|
|
|
|
if estimator.measurements != 3 {
|
|
|
|
|
|
t.Fatalf("measured beyond the first excess: %d measurements", estimator.measurements)
|
|
|
|
|
|
}
|
|
|
|
|
|
estimator.measurements = 0
|
2026-11-17 04:25:35 +00:00
|
|
|
|
fitted, err = fitValidationContext(item, 1, estimator, nil)
|
2026-11-16 14:14:21 +00:00
|
|
|
|
if err != nil || fitted.tokens <= 1 || estimator.measurements != 1 {
|
|
|
|
|
|
t.Fatalf("oversized required data: tokens=%d measurements=%d error=%v", fitted.tokens, estimator.measurements, err)
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
2026-11-17 04:25:35 +00:00
|
|
|
|
|
|
|
|
|
|
func TestValidationSkipsOptionalContextWithoutBudget(t *testing.T) {
|
|
|
|
|
|
item := LLMValidationItem{CorrectionIndex: 7, OriginalText: "teh", CorrectedText: "the"}
|
|
|
|
|
|
estimator := chunking.NewSimpleTokenEstimator()
|
|
|
|
|
|
required := serializedTokenCount(t, []LLMValidationItem{item})
|
|
|
|
|
|
loads := 0
|
|
|
|
|
|
optional := func(LLMValidationItem) ([]NeighboringSegment, []glossary.Entry) {
|
|
|
|
|
|
loads++
|
|
|
|
|
|
return nil, []glossary.Entry{{Name: "optional"}}
|
|
|
|
|
|
}
|
|
|
|
|
|
for _, budget := range []int{required - 1, required} {
|
|
|
|
|
|
batches, oversized, err := prepareValidationBatches([]LLMValidationItem{item}, budget, estimator, optional)
|
|
|
|
|
|
if err != nil || loads != 0 {
|
|
|
|
|
|
t.Fatalf("budget=%d loads=%d error=%v", budget, loads, err)
|
|
|
|
|
|
}
|
|
|
|
|
|
if budget < required {
|
|
|
|
|
|
if len(batches) != 0 || len(oversized) != 1 {
|
|
|
|
|
|
t.Fatalf("oversized result: batches=%+v rejected=%+v", batches, oversized)
|
|
|
|
|
|
}
|
|
|
|
|
|
} else if len(batches) != 1 || len(oversized) != 0 || batches[0].TokenCount != required {
|
|
|
|
|
|
t.Fatalf("exact fit: batches=%+v rejected=%+v", batches, oversized)
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|