diff --git a/docs/internal/validators.md b/docs/internal/validators.md index 8257581..4dc18d4 100644 --- a/docs/internal/validators.md +++ b/docs/internal/validators.md @@ -156,15 +156,12 @@ already clear from its own segment does not require such evidence. No audio, out-of-chunk context, or complete transcript is implied. Context is serialized inside the validation item and included in the existing -payload token estimate, including the target speaker. If one enriched item exceeds -the configured limit, optional neighbors are removed from the end first, then -glossary entries from the end, -until it fits or only required candidate data remains. Required before/after text -is never truncated. Existing oversized-candidate handling still applies if that -required data does not fit. The built-in estimator reuses encoded fragment -counts for fitting and batching. Context is supplemental, not a promise that every -relevant definition or neighboring utterance always fits. This uses the existing -payload estimator, not a new guarantee about the provider's full context window. +payload token estimate, including the target speaker. Fitting measures the +required data first, then considers glossary entries followed by neighboring +segments in order. It stops at the first entry that exceeds the budget. Required +before/after text is never truncated. A candidate whose required data exceeds +the budget receives an unavailable decision. The built-in estimator reuses +encoded fragment counts for fitting and batching. ## Protected Names diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index a116532..ac8234c 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -377,6 +377,13 @@ Audit #36. reduced measurements or allocation. Retain request-shape and available sibling expectations under over-budget input. +**Complete.** Validation budgets measure required data first and stop at the +first optional entry that does not fit, with glossary entries preceding +neighbors. An independent serialized-payload oracle checked fit and batching. +With 5,000 optional entries and room for one, allocations fell from 3.12 MB to +2.13 KB per operation (Go 1.27.3, twenty iterations). The CLI build and full +offline suite passed. + ## Stage 11 — Hand checked selections to execution Audit #24 (readiness), #37, #49 (module port), #50, #55, #56. diff --git a/internal/framework/validators/llm_batching.go b/internal/framework/validators/llm_batching.go index fb015db..33681ac 100644 --- a/internal/framework/validators/llm_batching.go +++ b/internal/framework/validators/llm_batching.go @@ -15,35 +15,35 @@ type LLMValidationBatch struct { } func prepareValidationBatches(items []LLMValidationItem, budget int, estimator chunking.TokenEstimator) ([]LLMValidationBatch, []Decision, error) { - if simple, ok := estimator.(*chunking.SimpleTokenEstimator); ok { - usable := make([]LLMValidationItem, 0, len(items)) - costs := make([]validationCost, 0, len(items)) - var oversized []Decision - for _, item := range items { - fitted, cost, err := fitSimpleValidationContext(item, budget, simple) - if err != nil { - return nil, oversized, err - } - if cost.arrayTokens() > budget { - oversized = append(oversized, oversizedValidationDecision(item.CorrectionIndex)) - } else { - usable = append(usable, fitted) - costs = append(costs, cost) - } + if estimator == nil { + estimator = chunking.NewSimpleTokenEstimator() + } + simple, _ := estimator.(*chunking.SimpleTokenEstimator) + usable := make([]LLMValidationItem, 0, len(items)) + var costs []validationCost + var sizes []int + var oversized []Decision + for _, item := range items { + fitted, err := fitValidationContext(item, budget, estimator) + if err != nil { + return nil, oversized, err } + if fitted.tokens > budget { + oversized = append(oversized, oversizedValidationDecision(item.CorrectionIndex)) + continue + } + usable = append(usable, fitted.item) + if simple != nil { + costs = append(costs, fitted.cost) + } else { + sizes = append(sizes, fitted.tokens) + } + } + if simple != nil { batches, err := chunkSimpleValidationItems(usable, costs, budget, simple) return batches, oversized, err } - - for i := range items { - items[i] = fitValidationContext(items[i], budget, estimator) - } - oversized := oversizedValidationDecisions(items, budget, estimator) - usable := filterItemsByDecision(items, oversized) - if len(usable) == 0 { - return nil, oversized, nil - } - batches, err := ChunkLLMValidationItems(usable, budget, estimator) + batches, err := chunkSerializedValidationItems(usable, sizes, budget, estimator) return batches, oversized, err } @@ -61,26 +61,40 @@ func ChunkLLMValidationItems(items []LLMValidationItem, maxPromptTokens int, est return chunkSimpleValidationItems(items, nil, maxPromptTokens, simple) } - batches := make([]LLMValidationBatch, 0) - current := make([]LLMValidationItem, 0) + return chunkSerializedValidationItems(items, nil, maxPromptTokens, estimator) +} + +func chunkSerializedValidationItems(items []LLMValidationItem, sizes []int, maxPromptTokens int, estimator chunking.TokenEstimator) ([]LLMValidationBatch, error) { + var batches []LLMValidationBatch + var current []LLMValidationItem currentTokens := 0 - for _, item := range items { - singleTokens, err := estimateBatchTokens(estimator, []LLMValidationItem{item}) - if err != nil { - return nil, err + for i, item := range items { + var singleTokens int + if sizes != nil { + singleTokens = sizes[i] + } else { + var err error + singleTokens, err = estimateBatchTokens(estimator, []LLMValidationItem{item}) + if err != nil { + return nil, err + } } if singleTokens > maxPromptTokens { return nil, fmt.Errorf("single validation proposal exceeds max prompt tokens") } + if len(current) == 0 { + current, currentTokens = []LLMValidationItem{item}, singleTokens + continue + } candidate := append(append([]LLMValidationItem(nil), current...), item) candidateTokens, err := estimateBatchTokens(estimator, candidate) if err != nil { return nil, err } - if len(current) > 0 && candidateTokens > maxPromptTokens { + if candidateTokens > maxPromptTokens { batches = append(batches, LLMValidationBatch{ BatchIndex: len(batches), Items: append([]LLMValidationItem(nil), current...), diff --git a/internal/framework/validators/llm_budget.go b/internal/framework/validators/llm_budget.go index b90ca1f..3134340 100644 --- a/internal/framework/validators/llm_budget.go +++ b/internal/framework/validators/llm_budget.go @@ -39,52 +39,70 @@ func measureValidationItem(estimator *chunking.SimpleTokenEstimator, item LLMVal return measureJSON(estimator, payload), nil } -func measureContextPrefixes[T any](estimator *chunking.SimpleTokenEstimator, field string, entries []T) ([]validationCost, error) { - prefixes := make([]validationCost, len(entries)+1) - cost := measureJSON(estimator, []byte(`,"`+field+`":[]`)) - for i, entry := range entries { - payload, err := json.Marshal(entry) - if err != nil { - return nil, fmt.Errorf("marshal validation %s: %w", field, err) - } - if i > 0 { - cost.punctuation++ // separator between entries - } - cost = cost.add(measureJSON(estimator, payload)) - prefixes[i+1] = cost - } - return prefixes, nil +type fittedValidationItem struct { + item LLMValidationItem + cost validationCost + tokens int } -func fitSimpleValidationContext(item LLMValidationItem, budget int, estimator *chunking.SimpleTokenEstimator) (LLMValidationItem, validationCost, error) { - required := item - required.NeighboringSegments, required.GlossaryEntries = nil, nil - base, err := measureValidationItem(estimator, required) - if err != nil { - return item, validationCost{}, err +// fitValidationContext tries the required payload followed by glossary entries +// and then neighboring segments. The first excess ends the usable prefix. +func fitValidationContext(item LLMValidationItem, budget int, estimator chunking.TokenEstimator) (fittedValidationItem, error) { + fitted := fittedValidationItem{item: item} + fitted.item.GlossaryEntries = item.GlossaryEntries[:0] + fitted.item.NeighboringSegments = item.NeighboringSegments[:0] + simple, _ := estimator.(*chunking.SimpleTokenEstimator) + var err error + if simple != nil { + fitted.cost, err = measureValidationItem(simple, fitted.item) + fitted.tokens = fitted.cost.arrayTokens() + } else { + fitted.tokens, err = estimateBatchTokens(estimator, []LLMValidationItem{fitted.item}) } - if len(item.NeighboringSegments)+len(item.GlossaryEntries) == 0 { - return item, base, nil + if err != nil || fitted.tokens > budget { + return fitted, err } - neighbors, err := measureContextPrefixes(estimator, "neighboring_segments", item.NeighboringSegments) - if err != nil { - return item, validationCost{}, err + try := func(candidate LLMValidationItem, field string, first bool, entry any) (bool, error) { + cost := fitted.cost + var tokens int + if simple != nil { + payload, err := json.Marshal(entry) + if err != nil { + return false, fmt.Errorf("marshal validation %s: %w", field, err) + } + if first { + cost = cost.add(measureJSON(simple, []byte(`,"`+field+`":[]`))) + } else { + cost.punctuation++ // separator between entries + } + cost = cost.add(measureJSON(simple, payload)) + tokens = cost.arrayTokens() + } else { + var err error + tokens, err = estimateBatchTokens(estimator, []LLMValidationItem{candidate}) + if err != nil { + return false, err + } + } + if tokens > budget { + return false, nil + } + fitted = fittedValidationItem{item: candidate, cost: cost, tokens: tokens} + return true, nil } - glossary, err := measureContextPrefixes(estimator, "glossary_entries", item.GlossaryEntries) - if err != nil { - return item, validationCost{}, err + for i, entry := range item.GlossaryEntries { + candidate := fitted.item + candidate.GlossaryEntries = item.GlossaryEntries[:i+1] + if ok, err := try(candidate, "glossary_entries", i == 0, entry); !ok { + return fitted, err + } } - n, g := len(item.NeighboringSegments), len(item.GlossaryEntries) - cost := base.add(neighbors[n]).add(glossary[g]) - for n > 0 && cost.arrayTokens() > budget { - n-- - cost = base.add(neighbors[n]).add(glossary[g]) + for i, entry := range item.NeighboringSegments { + candidate := fitted.item + candidate.NeighboringSegments = item.NeighboringSegments[:i+1] + if ok, err := try(candidate, "neighboring_segments", i == 0, entry); !ok { + return fitted, err + } } - for g > 0 && cost.arrayTokens() > budget { - g-- - cost = base.add(glossary[g]) - } - item.NeighboringSegments = item.NeighboringSegments[:n] - item.GlossaryEntries = item.GlossaryEntries[:g] - return item, cost, nil + return fitted, nil } diff --git a/internal/framework/validators/llm_budget_test.go b/internal/framework/validators/llm_budget_test.go index 1123a16..387fe9b 100644 --- a/internal/framework/validators/llm_budget_test.go +++ b/internal/framework/validators/llm_budget_test.go @@ -6,13 +6,70 @@ import ( "slices" "strings" "testing" + "unicode" "gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking" "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary" ) -// Wrapping the same estimator exercises budgeting through exact serialization. -type serializedBudgetEstimator struct{ *chunking.SimpleTokenEstimator } +// Count the serialized payload independently from the production token estimator. +func serializedTokenCount(t *testing.T, items []LLMValidationItem) int { + t.Helper() + local := slices.Clone(items) + for i := range local { + local[i].CorrectionIndex = i + 1 + } + payload, err := json.Marshal(local) + if err != nil { + t.Fatal(err) + } + text := string(payload) + punctuation := 0 + for _, r := range text { + if unicode.IsPunct(r) && r != '\'' && r != '-' && r != '_' { + punctuation++ + } + } + return len(strings.Fields(text)) + punctuation/2 +} + +// Check the same context priority by trimming from the end of serialized +// payloads. This does not use the production prefix fitter or batch builder. +func serializedBudgetOracle(t *testing.T, items []LLMValidationItem, budget int) ([]LLMValidationBatch, []int) { + t.Helper() + var batches []LLMValidationBatch + var oversized []int + var current []LLMValidationItem + closeBatch := func() { + batches = append(batches, LLMValidationBatch{BatchIndex: len(batches), Items: current, TokenCount: serializedTokenCount(t, current)}) + } + for _, item := range items { + for serializedTokenCount(t, []LLMValidationItem{item}) > budget { + if n := len(item.NeighboringSegments); n > 0 { + item.NeighboringSegments = item.NeighboringSegments[:n-1] + } else if n := len(item.GlossaryEntries); n > 0 { + item.GlossaryEntries = item.GlossaryEntries[:n-1] + } else { + break + } + } + if serializedTokenCount(t, []LLMValidationItem{item}) > budget { + oversized = append(oversized, item.CorrectionIndex) + continue + } + candidate := append(slices.Clone(current), item) + if len(current) > 0 && serializedTokenCount(t, candidate) > budget { + closeBatch() + current = []LLMValidationItem{item} + } else { + current = candidate + } + } + if len(current) > 0 { + closeBatch() + } + return batches, oversized +} func TestComposedValidationBudgetMatchesSerializedPayload(t *testing.T) { items := make([]LLMValidationItem, 12) @@ -38,17 +95,20 @@ func TestComposedValidationBudgetMatchesSerializedPayload(t *testing.T) { items[1].CorrectedSegmentText = strings.Repeat("longer candidate. ", 12) simple := chunking.NewSimpleTokenEstimator() - serialized := serializedBudgetEstimator{simple} - full, err := estimateBatchTokens(serialized, items) - if err != nil { - t.Fatal(err) - } + full := serializedTokenCount(t, items) for budget := 1; budget <= full; budget++ { - got, rejected, err := prepareValidationBatches(slices.Clone(items), budget, simple) - want, wantRejected, serializedErr := prepareValidationBatches(slices.Clone(items), budget, serialized) - if err != nil || serializedErr != nil || !reflect.DeepEqual(got, want) || !reflect.DeepEqual(rejected, wantRejected) { - t.Fatalf("budget %d: composed=%+v rejected=%+v err=%v; serialized=%+v rejected=%+v err=%v", - budget, got, rejected, err, want, wantRejected, serializedErr) + got, rejected, err := prepareValidationBatches(items, budget, simple) + want, wantRejected := serializedBudgetOracle(t, items, budget) + var rejectedIndices []int + for _, decision := range rejected { + if decision.Disposition != DispositionUnavailable || decision.ReasonCode != ReasonValidatorInputTooLarge { + t.Fatalf("unexpected over-budget decision: %+v", decision) + } + rejectedIndices = append(rejectedIndices, decision.ProposalIndex) + } + if err != nil || !reflect.DeepEqual(got, want) || !slices.Equal(rejectedIndices, wantRejected) { + t.Fatalf("budget %d: composed=%+v rejected=%+v err=%v; serialized=%+v rejected=%+v", + budget, got, rejected, err, want, wantRejected) } } @@ -73,3 +133,37 @@ func TestComposedValidationBudgetMatchesSerializedPayload(t *testing.T) { } } } + +type countingEstimator struct { + chunking.SimpleTokenEstimator + measurements int +} + +func (e *countingEstimator) EstimateTokens(text string) int { + e.measurements++ + return e.SimpleTokenEstimator.EstimateTokens(text) +} + +func TestValidationFitsRequiredDataBeforeScanningOptionalContext(t *testing.T) { + item := LLMValidationItem{CorrectionIndex: 7, OriginalText: "teh", CorrectedText: "the"} + first := glossary.Entry{ID: "keep", Name: "one"} + prefix := item + prefix.GlossaryEntries = []glossary.Entry{first} + budget := serializedTokenCount(t, []LLMValidationItem{prefix}) + item.GlossaryEntries = append([]glossary.Entry{first, {Description: strings.Repeat("large body ", 500)}}, make([]glossary.Entry, 5000)...) + item.NeighboringSegments = []NeighboringSegment{{Text: "neighbor"}} + estimator := &countingEstimator{} + fitted, err := fitValidationContext(item, budget, estimator) + if err != nil || len(fitted.item.GlossaryEntries) != 1 || fitted.item.GlossaryEntries[0].ID != first.ID || + len(fitted.item.NeighboringSegments) != 0 || fitted.tokens != budget { + t.Fatalf("incorrect usable prefix: %+v (%v)", fitted, err) + } + if estimator.measurements != 3 { + t.Fatalf("measured beyond the first excess: %d measurements", estimator.measurements) + } + estimator.measurements = 0 + fitted, err = fitValidationContext(item, 1, estimator) + if err != nil || fitted.tokens <= 1 || estimator.measurements != 1 { + t.Fatalf("oversized required data: tokens=%d measurements=%d error=%v", fitted.tokens, estimator.measurements, err) + } +} diff --git a/internal/framework/validators/llm_context.go b/internal/framework/validators/llm_context.go index e899612..c9249b7 100644 --- a/internal/framework/validators/llm_context.go +++ b/internal/framework/validators/llm_context.go @@ -3,7 +3,6 @@ package validators import ( "slices" - "gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking" "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" ) @@ -64,24 +63,6 @@ func (view glossaryContext) validationContext(segments []schema.Segment, positio return neighbors, entries } -// fitValidationContext drops optional context before treating required candidate -// text as oversized. Glossary definitions take priority over adjacent segments. -func fitValidationContext(item LLMValidationItem, budget int, estimator chunking.TokenEstimator) LLMValidationItem { - for { - tokens, err := estimateBatchTokens(estimator, []LLMValidationItem{item}) - if err != nil || tokens <= budget { - return item - } - if len(item.NeighboringSegments) > 0 { - item.NeighboringSegments = item.NeighboringSegments[:len(item.NeighboringSegments)-1] - } else if len(item.GlossaryEntries) > 0 { - item.GlossaryEntries = item.GlossaryEntries[:len(item.GlossaryEntries)-1] - } else { - return item - } - } -} - func neighboringSegment(position string, segment schema.Segment) NeighboringSegment { return NeighboringSegment{Position: position, Speaker: segment.Speaker, Text: segment.Text, Categories: segment.Categories} } diff --git a/internal/framework/validators/llm_context_test.go b/internal/framework/validators/llm_context_test.go index 715d5e1..75d48a2 100644 --- a/internal/framework/validators/llm_context_test.go +++ b/internal/framework/validators/llm_context_test.go @@ -73,7 +73,11 @@ func TestValidationContextFitsBudgetWithoutDiscardingCandidate(t *testing.T) { enriched := base enriched.NeighboringSegments = []NeighboringSegment{{Position: "following", Text: strings.Repeat("long context ", 300)}} enriched.GlossaryEntries = []glossary.Entry{{Name: "cat", Description: strings.Repeat("long definition ", 300)}} - fitted := fitValidationContext(enriched, required, estimator) + fit, err := fitValidationContext(enriched, required, estimator) + if err != nil { + t.Fatal(err) + } + fitted := fit.item tokens, err := estimateBatchTokens(estimator, []LLMValidationItem{fitted}) if err != nil || tokens > required || fitted.Speaker != base.Speaker || fitted.OriginalSegmentText != base.OriginalSegmentText || fitted.CorrectedSegmentText != base.CorrectedSegmentText { t.Fatalf("fitted: %+v tokens=%d err=%v", fitted, tokens, err) diff --git a/internal/framework/validators/llm_validators.go b/internal/framework/validators/llm_validators.go index d291027..e857f67 100644 --- a/internal/framework/validators/llm_validators.go +++ b/internal/framework/validators/llm_validators.go @@ -330,40 +330,10 @@ func reconcileLLMResponse(items []LLMValidationItem, response LLMValidationRespo return result } -func oversizedValidationDecisions(items []LLMValidationItem, maxPromptTokens int, estimator chunking.TokenEstimator) []Decision { - var out []Decision - for _, item := range items { - singleTokens, err := estimateBatchTokens(estimator, []LLMValidationItem{item}) - if err != nil || singleTokens <= maxPromptTokens { - continue - } - out = append(out, oversizedValidationDecision(item.CorrectionIndex)) - } - return out -} - func oversizedValidationDecision(index int) Decision { return unavailable(index, ReasonValidatorInputTooLarge, "validation input exceeds max prompt tokens") } -func filterItemsByDecision(items []LLMValidationItem, decisions []Decision) []LLMValidationItem { - if len(decisions) == 0 { - return append([]LLMValidationItem(nil), items...) - } - rejected := make(map[int]struct{}, len(decisions)) - for _, decision := range decisions { - rejected[decision.ProposalIndex] = struct{}{} - } - out := make([]LLMValidationItem, 0, len(items)) - for _, item := range items { - if _, ok := rejected[item.CorrectionIndex]; ok { - continue - } - out = append(out, item) - } - return out -} - func rejectBatch(items []LLMValidationItem, reasonCode string, message string) []Decision { out := make([]Decision, 0, len(items)) for _, item := range items {