Measure only the usable validation context prefix

This commit is contained in:
2026-11-16 14:14:21 +00:00
parent 0c144fd5a5
commit b1d1b8302c
8 changed files with 229 additions and 144 deletions

View File

@@ -156,15 +156,12 @@ already clear from its own segment does not require such evidence. No audio,
out-of-chunk context, or complete transcript is implied.
Context is serialized inside the validation item and included in the existing
payload token estimate, including the target speaker. If one enriched item exceeds
the configured limit, optional neighbors are removed from the end first, then
glossary entries from the end,
until it fits or only required candidate data remains. Required before/after text
is never truncated. Existing oversized-candidate handling still applies if that
required data does not fit. The built-in estimator reuses encoded fragment
counts for fitting and batching. Context is supplemental, not a promise that every
relevant definition or neighboring utterance always fits. This uses the existing
payload estimator, not a new guarantee about the provider's full context window.
payload token estimate, including the target speaker. Fitting measures the
required data first, then considers glossary entries followed by neighboring
segments in order. It stops at the first entry that exceeds the budget. Required
before/after text is never truncated. A candidate whose required data exceeds
the budget receives an unavailable decision. The built-in estimator reuses
encoded fragment counts for fitting and batching.
## Protected Names

View File

@@ -377,6 +377,13 @@ Audit #36.
reduced measurements or allocation. Retain request-shape and available
sibling expectations under over-budget input.
**Complete.** Validation budgets measure required data first and stop at the
first optional entry that does not fit, with glossary entries preceding
neighbors. An independent serialized-payload oracle checked fit and batching.
With 5,000 optional entries and room for one, allocations fell from 3.12 MB to
2.13 KB per operation (Go 1.27.3, twenty iterations). The CLI build and full
offline suite passed.
## Stage 11 — Hand checked selections to execution
Audit #24 (readiness), #37, #49 (module port), #50, #55, #56.

View File

@@ -15,35 +15,35 @@ type LLMValidationBatch struct {
}
func prepareValidationBatches(items []LLMValidationItem, budget int, estimator chunking.TokenEstimator) ([]LLMValidationBatch, []Decision, error) {
if simple, ok := estimator.(*chunking.SimpleTokenEstimator); ok {
usable := make([]LLMValidationItem, 0, len(items))
costs := make([]validationCost, 0, len(items))
var oversized []Decision
for _, item := range items {
fitted, cost, err := fitSimpleValidationContext(item, budget, simple)
if err != nil {
return nil, oversized, err
}
if cost.arrayTokens() > budget {
oversized = append(oversized, oversizedValidationDecision(item.CorrectionIndex))
} else {
usable = append(usable, fitted)
costs = append(costs, cost)
}
if estimator == nil {
estimator = chunking.NewSimpleTokenEstimator()
}
simple, _ := estimator.(*chunking.SimpleTokenEstimator)
usable := make([]LLMValidationItem, 0, len(items))
var costs []validationCost
var sizes []int
var oversized []Decision
for _, item := range items {
fitted, err := fitValidationContext(item, budget, estimator)
if err != nil {
return nil, oversized, err
}
if fitted.tokens > budget {
oversized = append(oversized, oversizedValidationDecision(item.CorrectionIndex))
continue
}
usable = append(usable, fitted.item)
if simple != nil {
costs = append(costs, fitted.cost)
} else {
sizes = append(sizes, fitted.tokens)
}
}
if simple != nil {
batches, err := chunkSimpleValidationItems(usable, costs, budget, simple)
return batches, oversized, err
}
for i := range items {
items[i] = fitValidationContext(items[i], budget, estimator)
}
oversized := oversizedValidationDecisions(items, budget, estimator)
usable := filterItemsByDecision(items, oversized)
if len(usable) == 0 {
return nil, oversized, nil
}
batches, err := ChunkLLMValidationItems(usable, budget, estimator)
batches, err := chunkSerializedValidationItems(usable, sizes, budget, estimator)
return batches, oversized, err
}
@@ -61,26 +61,40 @@ func ChunkLLMValidationItems(items []LLMValidationItem, maxPromptTokens int, est
return chunkSimpleValidationItems(items, nil, maxPromptTokens, simple)
}
batches := make([]LLMValidationBatch, 0)
current := make([]LLMValidationItem, 0)
return chunkSerializedValidationItems(items, nil, maxPromptTokens, estimator)
}
func chunkSerializedValidationItems(items []LLMValidationItem, sizes []int, maxPromptTokens int, estimator chunking.TokenEstimator) ([]LLMValidationBatch, error) {
var batches []LLMValidationBatch
var current []LLMValidationItem
currentTokens := 0
for _, item := range items {
singleTokens, err := estimateBatchTokens(estimator, []LLMValidationItem{item})
if err != nil {
return nil, err
for i, item := range items {
var singleTokens int
if sizes != nil {
singleTokens = sizes[i]
} else {
var err error
singleTokens, err = estimateBatchTokens(estimator, []LLMValidationItem{item})
if err != nil {
return nil, err
}
}
if singleTokens > maxPromptTokens {
return nil, fmt.Errorf("single validation proposal exceeds max prompt tokens")
}
if len(current) == 0 {
current, currentTokens = []LLMValidationItem{item}, singleTokens
continue
}
candidate := append(append([]LLMValidationItem(nil), current...), item)
candidateTokens, err := estimateBatchTokens(estimator, candidate)
if err != nil {
return nil, err
}
if len(current) > 0 && candidateTokens > maxPromptTokens {
if candidateTokens > maxPromptTokens {
batches = append(batches, LLMValidationBatch{
BatchIndex: len(batches),
Items: append([]LLMValidationItem(nil), current...),

View File

@@ -39,52 +39,70 @@ func measureValidationItem(estimator *chunking.SimpleTokenEstimator, item LLMVal
return measureJSON(estimator, payload), nil
}
func measureContextPrefixes[T any](estimator *chunking.SimpleTokenEstimator, field string, entries []T) ([]validationCost, error) {
prefixes := make([]validationCost, len(entries)+1)
cost := measureJSON(estimator, []byte(`,"`+field+`":[]`))
for i, entry := range entries {
payload, err := json.Marshal(entry)
if err != nil {
return nil, fmt.Errorf("marshal validation %s: %w", field, err)
}
if i > 0 {
cost.punctuation++ // separator between entries
}
cost = cost.add(measureJSON(estimator, payload))
prefixes[i+1] = cost
}
return prefixes, nil
type fittedValidationItem struct {
item LLMValidationItem
cost validationCost
tokens int
}
func fitSimpleValidationContext(item LLMValidationItem, budget int, estimator *chunking.SimpleTokenEstimator) (LLMValidationItem, validationCost, error) {
required := item
required.NeighboringSegments, required.GlossaryEntries = nil, nil
base, err := measureValidationItem(estimator, required)
if err != nil {
return item, validationCost{}, err
// fitValidationContext tries the required payload followed by glossary entries
// and then neighboring segments. The first excess ends the usable prefix.
func fitValidationContext(item LLMValidationItem, budget int, estimator chunking.TokenEstimator) (fittedValidationItem, error) {
fitted := fittedValidationItem{item: item}
fitted.item.GlossaryEntries = item.GlossaryEntries[:0]
fitted.item.NeighboringSegments = item.NeighboringSegments[:0]
simple, _ := estimator.(*chunking.SimpleTokenEstimator)
var err error
if simple != nil {
fitted.cost, err = measureValidationItem(simple, fitted.item)
fitted.tokens = fitted.cost.arrayTokens()
} else {
fitted.tokens, err = estimateBatchTokens(estimator, []LLMValidationItem{fitted.item})
}
if len(item.NeighboringSegments)+len(item.GlossaryEntries) == 0 {
return item, base, nil
if err != nil || fitted.tokens > budget {
return fitted, err
}
neighbors, err := measureContextPrefixes(estimator, "neighboring_segments", item.NeighboringSegments)
if err != nil {
return item, validationCost{}, err
try := func(candidate LLMValidationItem, field string, first bool, entry any) (bool, error) {
cost := fitted.cost
var tokens int
if simple != nil {
payload, err := json.Marshal(entry)
if err != nil {
return false, fmt.Errorf("marshal validation %s: %w", field, err)
}
if first {
cost = cost.add(measureJSON(simple, []byte(`,"`+field+`":[]`)))
} else {
cost.punctuation++ // separator between entries
}
cost = cost.add(measureJSON(simple, payload))
tokens = cost.arrayTokens()
} else {
var err error
tokens, err = estimateBatchTokens(estimator, []LLMValidationItem{candidate})
if err != nil {
return false, err
}
}
if tokens > budget {
return false, nil
}
fitted = fittedValidationItem{item: candidate, cost: cost, tokens: tokens}
return true, nil
}
glossary, err := measureContextPrefixes(estimator, "glossary_entries", item.GlossaryEntries)
if err != nil {
return item, validationCost{}, err
for i, entry := range item.GlossaryEntries {
candidate := fitted.item
candidate.GlossaryEntries = item.GlossaryEntries[:i+1]
if ok, err := try(candidate, "glossary_entries", i == 0, entry); !ok {
return fitted, err
}
}
n, g := len(item.NeighboringSegments), len(item.GlossaryEntries)
cost := base.add(neighbors[n]).add(glossary[g])
for n > 0 && cost.arrayTokens() > budget {
n--
cost = base.add(neighbors[n]).add(glossary[g])
for i, entry := range item.NeighboringSegments {
candidate := fitted.item
candidate.NeighboringSegments = item.NeighboringSegments[:i+1]
if ok, err := try(candidate, "neighboring_segments", i == 0, entry); !ok {
return fitted, err
}
}
for g > 0 && cost.arrayTokens() > budget {
g--
cost = base.add(glossary[g])
}
item.NeighboringSegments = item.NeighboringSegments[:n]
item.GlossaryEntries = item.GlossaryEntries[:g]
return item, cost, nil
return fitted, nil
}

View File

@@ -6,13 +6,70 @@ import (
"slices"
"strings"
"testing"
"unicode"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
)
// Wrapping the same estimator exercises budgeting through exact serialization.
type serializedBudgetEstimator struct{ *chunking.SimpleTokenEstimator }
// Count the serialized payload independently from the production token estimator.
func serializedTokenCount(t *testing.T, items []LLMValidationItem) int {
t.Helper()
local := slices.Clone(items)
for i := range local {
local[i].CorrectionIndex = i + 1
}
payload, err := json.Marshal(local)
if err != nil {
t.Fatal(err)
}
text := string(payload)
punctuation := 0
for _, r := range text {
if unicode.IsPunct(r) && r != '\'' && r != '-' && r != '_' {
punctuation++
}
}
return len(strings.Fields(text)) + punctuation/2
}
// Check the same context priority by trimming from the end of serialized
// payloads. This does not use the production prefix fitter or batch builder.
func serializedBudgetOracle(t *testing.T, items []LLMValidationItem, budget int) ([]LLMValidationBatch, []int) {
t.Helper()
var batches []LLMValidationBatch
var oversized []int
var current []LLMValidationItem
closeBatch := func() {
batches = append(batches, LLMValidationBatch{BatchIndex: len(batches), Items: current, TokenCount: serializedTokenCount(t, current)})
}
for _, item := range items {
for serializedTokenCount(t, []LLMValidationItem{item}) > budget {
if n := len(item.NeighboringSegments); n > 0 {
item.NeighboringSegments = item.NeighboringSegments[:n-1]
} else if n := len(item.GlossaryEntries); n > 0 {
item.GlossaryEntries = item.GlossaryEntries[:n-1]
} else {
break
}
}
if serializedTokenCount(t, []LLMValidationItem{item}) > budget {
oversized = append(oversized, item.CorrectionIndex)
continue
}
candidate := append(slices.Clone(current), item)
if len(current) > 0 && serializedTokenCount(t, candidate) > budget {
closeBatch()
current = []LLMValidationItem{item}
} else {
current = candidate
}
}
if len(current) > 0 {
closeBatch()
}
return batches, oversized
}
func TestComposedValidationBudgetMatchesSerializedPayload(t *testing.T) {
items := make([]LLMValidationItem, 12)
@@ -38,17 +95,20 @@ func TestComposedValidationBudgetMatchesSerializedPayload(t *testing.T) {
items[1].CorrectedSegmentText = strings.Repeat("longer candidate. ", 12)
simple := chunking.NewSimpleTokenEstimator()
serialized := serializedBudgetEstimator{simple}
full, err := estimateBatchTokens(serialized, items)
if err != nil {
t.Fatal(err)
}
full := serializedTokenCount(t, items)
for budget := 1; budget <= full; budget++ {
got, rejected, err := prepareValidationBatches(slices.Clone(items), budget, simple)
want, wantRejected, serializedErr := prepareValidationBatches(slices.Clone(items), budget, serialized)
if err != nil || serializedErr != nil || !reflect.DeepEqual(got, want) || !reflect.DeepEqual(rejected, wantRejected) {
t.Fatalf("budget %d: composed=%+v rejected=%+v err=%v; serialized=%+v rejected=%+v err=%v",
budget, got, rejected, err, want, wantRejected, serializedErr)
got, rejected, err := prepareValidationBatches(items, budget, simple)
want, wantRejected := serializedBudgetOracle(t, items, budget)
var rejectedIndices []int
for _, decision := range rejected {
if decision.Disposition != DispositionUnavailable || decision.ReasonCode != ReasonValidatorInputTooLarge {
t.Fatalf("unexpected over-budget decision: %+v", decision)
}
rejectedIndices = append(rejectedIndices, decision.ProposalIndex)
}
if err != nil || !reflect.DeepEqual(got, want) || !slices.Equal(rejectedIndices, wantRejected) {
t.Fatalf("budget %d: composed=%+v rejected=%+v err=%v; serialized=%+v rejected=%+v",
budget, got, rejected, err, want, wantRejected)
}
}
@@ -73,3 +133,37 @@ func TestComposedValidationBudgetMatchesSerializedPayload(t *testing.T) {
}
}
}
type countingEstimator struct {
chunking.SimpleTokenEstimator
measurements int
}
func (e *countingEstimator) EstimateTokens(text string) int {
e.measurements++
return e.SimpleTokenEstimator.EstimateTokens(text)
}
func TestValidationFitsRequiredDataBeforeScanningOptionalContext(t *testing.T) {
item := LLMValidationItem{CorrectionIndex: 7, OriginalText: "teh", CorrectedText: "the"}
first := glossary.Entry{ID: "keep", Name: "one"}
prefix := item
prefix.GlossaryEntries = []glossary.Entry{first}
budget := serializedTokenCount(t, []LLMValidationItem{prefix})
item.GlossaryEntries = append([]glossary.Entry{first, {Description: strings.Repeat("large body ", 500)}}, make([]glossary.Entry, 5000)...)
item.NeighboringSegments = []NeighboringSegment{{Text: "neighbor"}}
estimator := &countingEstimator{}
fitted, err := fitValidationContext(item, budget, estimator)
if err != nil || len(fitted.item.GlossaryEntries) != 1 || fitted.item.GlossaryEntries[0].ID != first.ID ||
len(fitted.item.NeighboringSegments) != 0 || fitted.tokens != budget {
t.Fatalf("incorrect usable prefix: %+v (%v)", fitted, err)
}
if estimator.measurements != 3 {
t.Fatalf("measured beyond the first excess: %d measurements", estimator.measurements)
}
estimator.measurements = 0
fitted, err = fitValidationContext(item, 1, estimator)
if err != nil || fitted.tokens <= 1 || estimator.measurements != 1 {
t.Fatalf("oversized required data: tokens=%d measurements=%d error=%v", fitted.tokens, estimator.measurements, err)
}
}

View File

@@ -3,7 +3,6 @@ package validators
import (
"slices"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
)
@@ -64,24 +63,6 @@ func (view glossaryContext) validationContext(segments []schema.Segment, positio
return neighbors, entries
}
// fitValidationContext drops optional context before treating required candidate
// text as oversized. Glossary definitions take priority over adjacent segments.
func fitValidationContext(item LLMValidationItem, budget int, estimator chunking.TokenEstimator) LLMValidationItem {
for {
tokens, err := estimateBatchTokens(estimator, []LLMValidationItem{item})
if err != nil || tokens <= budget {
return item
}
if len(item.NeighboringSegments) > 0 {
item.NeighboringSegments = item.NeighboringSegments[:len(item.NeighboringSegments)-1]
} else if len(item.GlossaryEntries) > 0 {
item.GlossaryEntries = item.GlossaryEntries[:len(item.GlossaryEntries)-1]
} else {
return item
}
}
}
func neighboringSegment(position string, segment schema.Segment) NeighboringSegment {
return NeighboringSegment{Position: position, Speaker: segment.Speaker, Text: segment.Text, Categories: segment.Categories}
}

View File

@@ -73,7 +73,11 @@ func TestValidationContextFitsBudgetWithoutDiscardingCandidate(t *testing.T) {
enriched := base
enriched.NeighboringSegments = []NeighboringSegment{{Position: "following", Text: strings.Repeat("long context ", 300)}}
enriched.GlossaryEntries = []glossary.Entry{{Name: "cat", Description: strings.Repeat("long definition ", 300)}}
fitted := fitValidationContext(enriched, required, estimator)
fit, err := fitValidationContext(enriched, required, estimator)
if err != nil {
t.Fatal(err)
}
fitted := fit.item
tokens, err := estimateBatchTokens(estimator, []LLMValidationItem{fitted})
if err != nil || tokens > required || fitted.Speaker != base.Speaker || fitted.OriginalSegmentText != base.OriginalSegmentText || fitted.CorrectedSegmentText != base.CorrectedSegmentText {
t.Fatalf("fitted: %+v tokens=%d err=%v", fitted, tokens, err)

View File

@@ -330,40 +330,10 @@ func reconcileLLMResponse(items []LLMValidationItem, response LLMValidationRespo
return result
}
func oversizedValidationDecisions(items []LLMValidationItem, maxPromptTokens int, estimator chunking.TokenEstimator) []Decision {
var out []Decision
for _, item := range items {
singleTokens, err := estimateBatchTokens(estimator, []LLMValidationItem{item})
if err != nil || singleTokens <= maxPromptTokens {
continue
}
out = append(out, oversizedValidationDecision(item.CorrectionIndex))
}
return out
}
func oversizedValidationDecision(index int) Decision {
return unavailable(index, ReasonValidatorInputTooLarge, "validation input exceeds max prompt tokens")
}
func filterItemsByDecision(items []LLMValidationItem, decisions []Decision) []LLMValidationItem {
if len(decisions) == 0 {
return append([]LLMValidationItem(nil), items...)
}
rejected := make(map[int]struct{}, len(decisions))
for _, decision := range decisions {
rejected[decision.ProposalIndex] = struct{}{}
}
out := make([]LLMValidationItem, 0, len(items))
for _, item := range items {
if _, ok := rejected[item.CorrectionIndex]; ok {
continue
}
out = append(out, item)
}
return out
}
func rejectBatch(items []LLMValidationItem, reasonCode string, message string) []Decision {
out := make([]Decision, 0, len(items))
for _, item := range items {