75 lines
2.7 KiB
Go
75 lines
2.7 KiB
Go
package validators
|
||
|
||
import (
|
||
"encoding/json"
|
||
"reflect"
|
||
"slices"
|
||
"strings"
|
||
"testing"
|
||
|
||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
|
||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
|
||
)
|
||
|
||
// Wrapping the same estimator exercises budgeting through exact serialization.
|
||
type serializedBudgetEstimator struct{ *chunking.SimpleTokenEstimator }
|
||
|
||
func TestComposedValidationBudgetMatchesSerializedPayload(t *testing.T) {
|
||
items := make([]LLMValidationItem, 12)
|
||
for i := range items {
|
||
items[i] = LLMValidationItem{
|
||
CorrectionIndex: 100 + i*7,
|
||
OriginalText: "one\u00a0two!",
|
||
CorrectedText: "one two",
|
||
OriginalSegmentText: "she said \"one\u00a0two!\"\n",
|
||
CorrectedSegmentText: "she said <one & two> — there.",
|
||
Categories: []string{"say, then write", "repeat"},
|
||
}
|
||
}
|
||
items[0].NeighboringSegments = []NeighboringSegment{
|
||
{Position: "preceding", Text: "spaces\tand escapes\\", Categories: []string{"speaker line"}},
|
||
{Position: "following", Text: "‘quoted’\u2028words"},
|
||
}
|
||
items[0].GlossaryEntries = []glossary.Entry{
|
||
{ID: "name", Name: "Name?", Aliases: []string{"N. M.", "Name's"}},
|
||
{ID: "two", Name: "One Two", Description: "some\u2003meaning…"},
|
||
}
|
||
items[1].CorrectedSegmentText = strings.Repeat("longer candidate. ", 12)
|
||
|
||
simple := chunking.NewSimpleTokenEstimator()
|
||
serialized := serializedBudgetEstimator{simple}
|
||
full, err := estimateBatchTokens(serialized, items)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
for budget := 1; budget <= full; budget++ {
|
||
got, rejected, err := prepareValidationBatches(slices.Clone(items), budget, simple)
|
||
want, wantRejected, serializedErr := prepareValidationBatches(slices.Clone(items), budget, serialized)
|
||
if err != nil || serializedErr != nil || !reflect.DeepEqual(got, want) || !reflect.DeepEqual(rejected, wantRejected) {
|
||
t.Fatalf("budget %d: composed=%+v rejected=%+v err=%v; serialized=%+v rejected=%+v err=%v",
|
||
budget, got, rejected, err, want, wantRejected, serializedErr)
|
||
}
|
||
}
|
||
|
||
batches, err := ChunkLLMValidationItems(items, full, simple)
|
||
if err != nil || len(batches) != 1 || batches[0].TokenCount != full {
|
||
t.Fatalf("full batch: %+v, %v", batches, err)
|
||
}
|
||
variables, err := BuildValidationVariables(batches[0].Items, "")
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
var payload []LLMValidationItem
|
||
if err := json.Unmarshal([]byte(variables["PayloadJSON"]), &payload); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if len(payload) != len(items) {
|
||
t.Fatalf("sent %d candidates, want %d", len(payload), len(items))
|
||
}
|
||
for i := range payload {
|
||
if payload[i].CorrectionIndex != i+1 || batches[0].Items[i].CorrectionIndex != items[i].CorrectionIndex {
|
||
t.Fatalf("candidate %d lost its local or run reference", i)
|
||
}
|
||
}
|
||
}
|