package validators import ( "bytes" "encoding/json" "slices" "strings" "testing" "unicode" "gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking" "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary" ) // Count the serialized payload independently from the production token estimator. func serializedTokenCount(t *testing.T, items []LLMValidationItem) int { t.Helper() local := slices.Clone(items) for i := range local { local[i].CorrectionIndex = i + 1 } payload, err := json.Marshal(local) if err != nil { t.Fatal(err) } text := string(payload) punctuation := 0 for _, r := range text { if unicode.IsPunct(r) && r != '\'' && r != '-' && r != '_' { punctuation++ } } return len(strings.Fields(text)) + punctuation/2 } // Check the same context priority by trimming from the end of serialized // payloads. This does not use the production prefix fitter or batch builder. func serializedBudgetOracle(t *testing.T, items []LLMValidationItem, budget int) ([]LLMValidationBatch, []int) { t.Helper() var batches []LLMValidationBatch var oversized []int var current []LLMValidationItem closeBatch := func() { batches = append(batches, LLMValidationBatch{BatchIndex: len(batches), Items: current, TokenCount: serializedTokenCount(t, current)}) } for _, item := range items { for serializedTokenCount(t, []LLMValidationItem{item}) > budget { if n := len(item.NeighboringSegments); n > 0 { item.NeighboringSegments = item.NeighboringSegments[:n-1] } else if n := len(item.GlossaryEntries); n > 0 { item.GlossaryEntries = item.GlossaryEntries[:n-1] } else { break } } if serializedTokenCount(t, []LLMValidationItem{item}) > budget { oversized = append(oversized, item.CorrectionIndex) continue } candidate := append(slices.Clone(current), item) if len(current) > 0 && serializedTokenCount(t, candidate) > budget { closeBatch() current = []LLMValidationItem{item} } else { current = candidate } } if len(current) > 0 { closeBatch() } return batches, oversized } func TestComposedValidationBudgetMatchesSerializedPayload(t *testing.T) { items := make([]LLMValidationItem, 12) for i := range items { items[i] = LLMValidationItem{ CorrectionIndex: 100 + i*7, Speaker: "Guest \"A\" — session host", OriginalText: "one\u00a0two!", CorrectedText: "one two", OriginalSegmentText: "she said \"one\u00a0two!\"\n", CorrectedSegmentText: "she said — there.", Categories: []string{"say, then write", "repeat"}, } } items[0].NeighboringSegments = []NeighboringSegment{ {Position: "preceding", Text: "spaces\tand escapes\\", Categories: []string{"speaker line"}}, {Position: "following", Text: "‘quoted’\u2028words"}, } items[0].GlossaryEntries = []glossary.Entry{ {ID: "name", Name: "Name?", Aliases: []string{"N. M.", "Name's"}}, {ID: "two", Name: "One Two", Description: "some\u2003meaning…"}, } items[1].CorrectedSegmentText = strings.Repeat("longer candidate. ", 12) simple := chunking.NewSimpleTokenEstimator() full := serializedTokenCount(t, items) for budget := 1; budget <= full; budget++ { got, rejected, err := prepareValidationBatches(items, budget, simple, nil) want, wantRejected := serializedBudgetOracle(t, items, budget) var rejectedIndices []int for _, decision := range rejected { if decision.Disposition != DispositionUnavailable || decision.ReasonCode != ReasonValidatorInputTooLarge { t.Fatalf("unexpected over-budget decision: %+v", decision) } rejectedIndices = append(rejectedIndices, decision.ProposalIndex) } gotJSON, gotErr := json.Marshal(got) wantJSON, wantErr := json.Marshal(want) if err != nil || gotErr != nil || wantErr != nil || !bytes.Equal(gotJSON, wantJSON) || !slices.Equal(rejectedIndices, wantRejected) { t.Fatalf("budget %d: composed=%+v rejected=%+v err=%v; serialized=%+v rejected=%+v", budget, got, rejected, err, want, wantRejected) } } batches, err := ChunkLLMValidationItems(items, full, simple) if err != nil || len(batches) != 1 || batches[0].TokenCount != full { t.Fatalf("full batch: %+v, %v", batches, err) } variables, err := BuildValidationVariables(batches[0].Items, "") if err != nil { t.Fatal(err) } var payload []LLMValidationItem if err := json.Unmarshal([]byte(variables["PayloadJSON"]), &payload); err != nil { t.Fatal(err) } if len(payload) != len(items) { t.Fatalf("sent %d candidates, want %d", len(payload), len(items)) } for i := range payload { if payload[i].CorrectionIndex != i+1 || batches[0].Items[i].CorrectionIndex != items[i].CorrectionIndex { t.Fatalf("candidate %d lost its local or run reference", i) } } } type countingEstimator struct { chunking.SimpleTokenEstimator measurements int } func (e *countingEstimator) EstimateTokens(text string) int { e.measurements++ return e.SimpleTokenEstimator.EstimateTokens(text) } func TestValidationFitsRequiredDataBeforeScanningOptionalContext(t *testing.T) { item := LLMValidationItem{CorrectionIndex: 7, OriginalText: "teh", CorrectedText: "the"} first := glossary.Entry{ID: "keep", Name: "one"} prefix := item prefix.GlossaryEntries = []glossary.Entry{first} budget := serializedTokenCount(t, []LLMValidationItem{prefix}) item.GlossaryEntries = append([]glossary.Entry{first, {Description: strings.Repeat("large body ", 500)}}, make([]glossary.Entry, 5000)...) item.NeighboringSegments = []NeighboringSegment{{Text: "neighbor"}} estimator := &countingEstimator{} fitted, err := fitValidationContext(item, budget, estimator, nil) if err != nil || len(fitted.item.GlossaryEntries) != 1 || fitted.item.GlossaryEntries[0].ID != first.ID || len(fitted.item.NeighboringSegments) != 0 || fitted.tokens != budget { t.Fatalf("incorrect usable prefix: %+v (%v)", fitted, err) } if estimator.measurements != 3 { t.Fatalf("measured beyond the first excess: %d measurements", estimator.measurements) } estimator.measurements = 0 fitted, err = fitValidationContext(item, 1, estimator, nil) if err != nil || fitted.tokens <= 1 || estimator.measurements != 1 { t.Fatalf("oversized required data: tokens=%d measurements=%d error=%v", fitted.tokens, estimator.measurements, err) } } func TestValidationSkipsOptionalContextWithoutBudget(t *testing.T) { item := LLMValidationItem{CorrectionIndex: 7, OriginalText: "teh", CorrectedText: "the"} estimator := chunking.NewSimpleTokenEstimator() required := serializedTokenCount(t, []LLMValidationItem{item}) loads := 0 optional := func(LLMValidationItem) ([]NeighboringSegment, []glossary.Entry) { loads++ return nil, []glossary.Entry{{Name: "optional"}} } for _, budget := range []int{required - 1, required} { batches, oversized, err := prepareValidationBatches([]LLMValidationItem{item}, budget, estimator, optional) if err != nil || loads != 0 { t.Fatalf("budget=%d loads=%d error=%v", budget, loads, err) } if budget < required { if len(batches) != 0 || len(oversized) != 1 { t.Fatalf("oversized result: batches=%+v rejected=%+v", batches, oversized) } } else if len(batches) != 1 || len(oversized) != 0 || batches[0].TokenCount != required { t.Fatalf("exact fit: batches=%+v rejected=%+v", batches, oversized) } } }