170 lines
6.0 KiB
Go
170 lines
6.0 KiB
Go
package validators
|
||
|
||
import (
|
||
"encoding/json"
|
||
"reflect"
|
||
"slices"
|
||
"strings"
|
||
"testing"
|
||
"unicode"
|
||
|
||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
|
||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
|
||
)
|
||
|
||
// Count the serialized payload independently from the production token estimator.
|
||
func serializedTokenCount(t *testing.T, items []LLMValidationItem) int {
|
||
t.Helper()
|
||
local := slices.Clone(items)
|
||
for i := range local {
|
||
local[i].CorrectionIndex = i + 1
|
||
}
|
||
payload, err := json.Marshal(local)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
text := string(payload)
|
||
punctuation := 0
|
||
for _, r := range text {
|
||
if unicode.IsPunct(r) && r != '\'' && r != '-' && r != '_' {
|
||
punctuation++
|
||
}
|
||
}
|
||
return len(strings.Fields(text)) + punctuation/2
|
||
}
|
||
|
||
// Check the same context priority by trimming from the end of serialized
|
||
// payloads. This does not use the production prefix fitter or batch builder.
|
||
func serializedBudgetOracle(t *testing.T, items []LLMValidationItem, budget int) ([]LLMValidationBatch, []int) {
|
||
t.Helper()
|
||
var batches []LLMValidationBatch
|
||
var oversized []int
|
||
var current []LLMValidationItem
|
||
closeBatch := func() {
|
||
batches = append(batches, LLMValidationBatch{BatchIndex: len(batches), Items: current, TokenCount: serializedTokenCount(t, current)})
|
||
}
|
||
for _, item := range items {
|
||
for serializedTokenCount(t, []LLMValidationItem{item}) > budget {
|
||
if n := len(item.NeighboringSegments); n > 0 {
|
||
item.NeighboringSegments = item.NeighboringSegments[:n-1]
|
||
} else if n := len(item.GlossaryEntries); n > 0 {
|
||
item.GlossaryEntries = item.GlossaryEntries[:n-1]
|
||
} else {
|
||
break
|
||
}
|
||
}
|
||
if serializedTokenCount(t, []LLMValidationItem{item}) > budget {
|
||
oversized = append(oversized, item.CorrectionIndex)
|
||
continue
|
||
}
|
||
candidate := append(slices.Clone(current), item)
|
||
if len(current) > 0 && serializedTokenCount(t, candidate) > budget {
|
||
closeBatch()
|
||
current = []LLMValidationItem{item}
|
||
} else {
|
||
current = candidate
|
||
}
|
||
}
|
||
if len(current) > 0 {
|
||
closeBatch()
|
||
}
|
||
return batches, oversized
|
||
}
|
||
|
||
func TestComposedValidationBudgetMatchesSerializedPayload(t *testing.T) {
|
||
items := make([]LLMValidationItem, 12)
|
||
for i := range items {
|
||
items[i] = LLMValidationItem{
|
||
CorrectionIndex: 100 + i*7,
|
||
Speaker: "Guest \"A\" — session host",
|
||
OriginalText: "one\u00a0two!",
|
||
CorrectedText: "one two",
|
||
OriginalSegmentText: "she said \"one\u00a0two!\"\n",
|
||
CorrectedSegmentText: "she said <one & two> — there.",
|
||
Categories: []string{"say, then write", "repeat"},
|
||
}
|
||
}
|
||
items[0].NeighboringSegments = []NeighboringSegment{
|
||
{Position: "preceding", Text: "spaces\tand escapes\\", Categories: []string{"speaker line"}},
|
||
{Position: "following", Text: "‘quoted’\u2028words"},
|
||
}
|
||
items[0].GlossaryEntries = []glossary.Entry{
|
||
{ID: "name", Name: "Name?", Aliases: []string{"N. M.", "Name's"}},
|
||
{ID: "two", Name: "One Two", Description: "some\u2003meaning…"},
|
||
}
|
||
items[1].CorrectedSegmentText = strings.Repeat("longer candidate. ", 12)
|
||
|
||
simple := chunking.NewSimpleTokenEstimator()
|
||
full := serializedTokenCount(t, items)
|
||
for budget := 1; budget <= full; budget++ {
|
||
got, rejected, err := prepareValidationBatches(items, budget, simple)
|
||
want, wantRejected := serializedBudgetOracle(t, items, budget)
|
||
var rejectedIndices []int
|
||
for _, decision := range rejected {
|
||
if decision.Disposition != DispositionUnavailable || decision.ReasonCode != ReasonValidatorInputTooLarge {
|
||
t.Fatalf("unexpected over-budget decision: %+v", decision)
|
||
}
|
||
rejectedIndices = append(rejectedIndices, decision.ProposalIndex)
|
||
}
|
||
if err != nil || !reflect.DeepEqual(got, want) || !slices.Equal(rejectedIndices, wantRejected) {
|
||
t.Fatalf("budget %d: composed=%+v rejected=%+v err=%v; serialized=%+v rejected=%+v",
|
||
budget, got, rejected, err, want, wantRejected)
|
||
}
|
||
}
|
||
|
||
batches, err := ChunkLLMValidationItems(items, full, simple)
|
||
if err != nil || len(batches) != 1 || batches[0].TokenCount != full {
|
||
t.Fatalf("full batch: %+v, %v", batches, err)
|
||
}
|
||
variables, err := BuildValidationVariables(batches[0].Items, "")
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
var payload []LLMValidationItem
|
||
if err := json.Unmarshal([]byte(variables["PayloadJSON"]), &payload); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if len(payload) != len(items) {
|
||
t.Fatalf("sent %d candidates, want %d", len(payload), len(items))
|
||
}
|
||
for i := range payload {
|
||
if payload[i].CorrectionIndex != i+1 || batches[0].Items[i].CorrectionIndex != items[i].CorrectionIndex {
|
||
t.Fatalf("candidate %d lost its local or run reference", i)
|
||
}
|
||
}
|
||
}
|
||
|
||
type countingEstimator struct {
|
||
chunking.SimpleTokenEstimator
|
||
measurements int
|
||
}
|
||
|
||
func (e *countingEstimator) EstimateTokens(text string) int {
|
||
e.measurements++
|
||
return e.SimpleTokenEstimator.EstimateTokens(text)
|
||
}
|
||
|
||
func TestValidationFitsRequiredDataBeforeScanningOptionalContext(t *testing.T) {
|
||
item := LLMValidationItem{CorrectionIndex: 7, OriginalText: "teh", CorrectedText: "the"}
|
||
first := glossary.Entry{ID: "keep", Name: "one"}
|
||
prefix := item
|
||
prefix.GlossaryEntries = []glossary.Entry{first}
|
||
budget := serializedTokenCount(t, []LLMValidationItem{prefix})
|
||
item.GlossaryEntries = append([]glossary.Entry{first, {Description: strings.Repeat("large body ", 500)}}, make([]glossary.Entry, 5000)...)
|
||
item.NeighboringSegments = []NeighboringSegment{{Text: "neighbor"}}
|
||
estimator := &countingEstimator{}
|
||
fitted, err := fitValidationContext(item, budget, estimator)
|
||
if err != nil || len(fitted.item.GlossaryEntries) != 1 || fitted.item.GlossaryEntries[0].ID != first.ID ||
|
||
len(fitted.item.NeighboringSegments) != 0 || fitted.tokens != budget {
|
||
t.Fatalf("incorrect usable prefix: %+v (%v)", fitted, err)
|
||
}
|
||
if estimator.measurements != 3 {
|
||
t.Fatalf("measured beyond the first excess: %d measurements", estimator.measurements)
|
||
}
|
||
estimator.measurements = 0
|
||
fitted, err = fitValidationContext(item, 1, estimator)
|
||
if err != nil || fitted.tokens <= 1 || estimator.measurements != 1 {
|
||
t.Fatalf("oversized required data: tokens=%d measurements=%d error=%v", fitted.tokens, estimator.measurements, err)
|
||
}
|
||
}
|