Collect optional validation context only when it can fit

This commit is contained in:
2026-11-17 04:25:35 +00:00
parent 44c767f285
commit d26991f0bf
9 changed files with 128 additions and 56 deletions

View File

@@ -331,6 +331,11 @@ form. Retain existing handling for a mismatched repair reference.
### R10 — Fit Required Validation Data Before Collecting Optional Context (P3)
**Complete.** Fitting checks required data before loading glossary and
neighbor context. Context selection reuses source matches within a validator
request and matches each revision independently. Tests verify skipped loading,
candidate-specific context, and equivalence with the serialized budget oracle.
**Affected plan:** Stages 8 and 10.
**Evidence:** [Item construction](../../internal/framework/validators/llm_validators.go#L251)

View File

@@ -14,7 +14,7 @@ type LLMValidationBatch struct {
TokenCount int `json:"token_count"`
}
func prepareValidationBatches(items []LLMValidationItem, budget int, estimator chunking.TokenEstimator) ([]LLMValidationBatch, []Decision, error) {
func prepareValidationBatches(items []LLMValidationItem, budget int, estimator chunking.TokenEstimator, context optionalValidationContext) ([]LLMValidationBatch, []Decision, error) {
if estimator == nil {
estimator = chunking.NewSimpleTokenEstimator()
}
@@ -24,7 +24,7 @@ func prepareValidationBatches(items []LLMValidationItem, budget int, estimator c
var sizes []int
var oversized []Decision
for _, item := range items {
fitted, err := fitValidationContext(item, budget, estimator)
fitted, err := fitValidationContext(item, budget, estimator, context)
if err != nil {
return nil, oversized, err
}

View File

@@ -3,6 +3,7 @@ package validators
import (
"encoding/json"
"fmt"
"slices"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
)
@@ -47,21 +48,31 @@ type fittedValidationItem struct {
// fitValidationContext tries the required payload followed by glossary entries
// and then neighboring segments. The first excess ends the usable prefix.
func fitValidationContext(item LLMValidationItem, budget int, estimator chunking.TokenEstimator) (fittedValidationItem, error) {
func fitValidationContext(item LLMValidationItem, budget int, estimator chunking.TokenEstimator, context optionalValidationContext) (result fittedValidationItem, err error) {
fitted := fittedValidationItem{item: item}
fitted.item.GlossaryEntries = item.GlossaryEntries[:0]
fitted.item.NeighboringSegments = item.NeighboringSegments[:0]
fitted.item.GlossaryEntries = nil
fitted.item.NeighboringSegments = nil
simple, _ := estimator.(*chunking.SimpleTokenEstimator)
var err error
if simple != nil {
fitted.cost, err = measureValidationItem(simple, fitted.item)
fitted.tokens = fitted.cost.arrayTokens()
} else {
fitted.tokens, err = estimateBatchTokens(estimator, []LLMValidationItem{fitted.item})
}
if err != nil || fitted.tokens > budget {
if err != nil || fitted.tokens > budget || (simple != nil && fitted.tokens == budget) {
return fitted, err
}
if context != nil {
item.NeighboringSegments, item.GlossaryEntries = context(item)
}
defer func() {
if len(result.item.GlossaryEntries) < len(item.GlossaryEntries) {
result.item.GlossaryEntries = slices.Clone(result.item.GlossaryEntries)
}
if len(result.item.NeighboringSegments) < len(item.NeighboringSegments) {
result.item.NeighboringSegments = slices.Clone(result.item.NeighboringSegments)
}
}()
try := func(candidate LLMValidationItem, field string, first bool, entry any) (bool, error) {
cost := fitted.cost
var tokens int

View File

@@ -1,8 +1,8 @@
package validators
import (
"bytes"
"encoding/json"
"reflect"
"slices"
"strings"
"testing"
@@ -97,7 +97,7 @@ func TestComposedValidationBudgetMatchesSerializedPayload(t *testing.T) {
simple := chunking.NewSimpleTokenEstimator()
full := serializedTokenCount(t, items)
for budget := 1; budget <= full; budget++ {
got, rejected, err := prepareValidationBatches(items, budget, simple)
got, rejected, err := prepareValidationBatches(items, budget, simple, nil)
want, wantRejected := serializedBudgetOracle(t, items, budget)
var rejectedIndices []int
for _, decision := range rejected {
@@ -106,7 +106,9 @@ func TestComposedValidationBudgetMatchesSerializedPayload(t *testing.T) {
}
rejectedIndices = append(rejectedIndices, decision.ProposalIndex)
}
if err != nil || !reflect.DeepEqual(got, want) || !slices.Equal(rejectedIndices, wantRejected) {
gotJSON, gotErr := json.Marshal(got)
wantJSON, wantErr := json.Marshal(want)
if err != nil || gotErr != nil || wantErr != nil || !bytes.Equal(gotJSON, wantJSON) || !slices.Equal(rejectedIndices, wantRejected) {
t.Fatalf("budget %d: composed=%+v rejected=%+v err=%v; serialized=%+v rejected=%+v",
budget, got, rejected, err, want, wantRejected)
}
@@ -153,7 +155,7 @@ func TestValidationFitsRequiredDataBeforeScanningOptionalContext(t *testing.T) {
item.GlossaryEntries = append([]glossary.Entry{first, {Description: strings.Repeat("large body ", 500)}}, make([]glossary.Entry, 5000)...)
item.NeighboringSegments = []NeighboringSegment{{Text: "neighbor"}}
estimator := &countingEstimator{}
fitted, err := fitValidationContext(item, budget, estimator)
fitted, err := fitValidationContext(item, budget, estimator, nil)
if err != nil || len(fitted.item.GlossaryEntries) != 1 || fitted.item.GlossaryEntries[0].ID != first.ID ||
len(fitted.item.NeighboringSegments) != 0 || fitted.tokens != budget {
t.Fatalf("incorrect usable prefix: %+v (%v)", fitted, err)
@@ -162,8 +164,32 @@ func TestValidationFitsRequiredDataBeforeScanningOptionalContext(t *testing.T) {
t.Fatalf("measured beyond the first excess: %d measurements", estimator.measurements)
}
estimator.measurements = 0
fitted, err = fitValidationContext(item, 1, estimator)
fitted, err = fitValidationContext(item, 1, estimator, nil)
if err != nil || fitted.tokens <= 1 || estimator.measurements != 1 {
t.Fatalf("oversized required data: tokens=%d measurements=%d error=%v", fitted.tokens, estimator.measurements, err)
}
}
func TestValidationSkipsOptionalContextWithoutBudget(t *testing.T) {
item := LLMValidationItem{CorrectionIndex: 7, OriginalText: "teh", CorrectedText: "the"}
estimator := chunking.NewSimpleTokenEstimator()
required := serializedTokenCount(t, []LLMValidationItem{item})
loads := 0
optional := func(LLMValidationItem) ([]NeighboringSegment, []glossary.Entry) {
loads++
return nil, []glossary.Entry{{Name: "optional"}}
}
for _, budget := range []int{required - 1, required} {
batches, oversized, err := prepareValidationBatches([]LLMValidationItem{item}, budget, estimator, optional)
if err != nil || loads != 0 {
t.Fatalf("budget=%d loads=%d error=%v", budget, loads, err)
}
if budget < required {
if len(batches) != 0 || len(oversized) != 1 {
t.Fatalf("oversized result: batches=%+v rejected=%+v", batches, oversized)
}
} else if len(batches) != 1 || len(oversized) != 0 || batches[0].TokenCount != required {
t.Fatalf("exact fit: batches=%+v rejected=%+v", batches, oversized)
}
}
}

View File

@@ -33,34 +33,58 @@ func prepareGlossaryContext(entries []glossary.Entry) glossaryContext {
return glossaryContext{entries: entries, positions: positions, vocabulary: glossary.NewProtectedVocabulary(glossary.Vocabulary(entries, nil))}
}
// validationContext supplies adjacent source segments within the current chunk
// and glossary definitions mentioned in the candidate or that local context.
// These fields travel in each item so batching accounts for their size.
func (view glossaryContext) validationContext(segments []schema.Segment, position int, corrected string) ([]NeighboringSegment, []glossary.Entry) {
var neighbors []NeighboringSegment
source := segments[position].Text
if position > 0 {
neighbors = append(neighbors, neighboringSegment("preceding", segments[position-1]))
}
if position+1 < len(segments) {
neighbors = append(neighbors, neighboringSegment("following", segments[position+1]))
}
type optionalValidationContext func(LLMValidationItem) ([]NeighboringSegment, []glossary.Entry)
texts := []string{source, corrected}
for _, neighbor := range neighbors {
texts = append(texts, neighbor.Text)
// validationContextFor builds optional context on demand. Unchanged segment
// matches are shared within the request. Corrected text is candidate-specific.
func validationContextFor(req Request) optionalValidationContext {
segments := req.sectionLookup().segments
var view *glossaryContext
if req.GlossaryView != nil {
view = &req.GlossaryView.context
}
mentioned := view.vocabulary.MentionedIdentities(texts...)
var matched []int
for identity := range mentioned {
matched = append(matched, view.positions[identity]...)
var sourceMentions map[int]map[glossary.Identity]struct{}
return func(item LLMValidationItem) ([]NeighboringSegment, []glossary.Entry) {
position := item.segmentPosition
var neighbors []NeighboringSegment
if position > 0 {
neighbors = append(neighbors, neighboringSegment("preceding", segments[position-1]))
}
if position+1 < len(segments) {
neighbors = append(neighbors, neighboringSegment("following", segments[position+1]))
}
if view == nil {
prepared := prepareGlossaryContext(req.Glossary)
view = &prepared
}
if view.vocabulary.Empty() {
return neighbors, nil
}
if sourceMentions == nil {
sourceMentions = make(map[int]map[glossary.Identity]struct{})
}
mentioned := view.vocabulary.MentionedIdentities(item.CorrectedSegmentText)
for i := max(0, position-1); i < min(len(segments), position+2); i++ {
identities, ok := sourceMentions[i]
if !ok {
identities = view.vocabulary.MentionedIdentities(segments[i].Text)
sourceMentions[i] = identities
}
for id := range identities {
mentioned[id] = struct{}{}
}
}
var matched []int
for identity := range mentioned {
matched = append(matched, view.positions[identity]...)
}
slices.Sort(matched)
var entries []glossary.Entry
for _, index := range matched {
entries = append(entries, view.entries[index])
}
return neighbors, entries
}
slices.Sort(matched)
var entries []glossary.Entry
for _, index := range matched {
entries = append(entries, view.entries[index])
}
return neighbors, entries
}
func neighboringSegment(position string, segment schema.Segment) NeighboringSegment {

View File

@@ -11,6 +11,16 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
)
func buildContextItems(req Request) ([]LLMValidationItem, []Decision) {
req.Section = req.sectionLookup()
items, immediate := buildValidationItems(req)
context := validationContextFor(req)
for i := range items {
items[i].NeighboringSegments, items[i].GlossaryEntries = context(items[i])
}
return items, immediate
}
func TestValidationContextCarriesAdjacentEvidenceAndRelevantDefinitions(t *testing.T) {
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(7, "gestures", "Jesters")})
req.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{
@@ -25,7 +35,7 @@ func TestValidationContextCarriesAdjacentEvidenceAndRelevantDefinitions(t *testi
{ID: "lyra", Name: "Lyra", Description: "The host"},
{ID: "unrelated", Name: "Unrelated", Description: "Not mentioned"},
}
payload, immediate := buildValidationItems(req)
payload, immediate := buildContextItems(req)
if len(immediate) != 0 || len(payload) != 1 {
t.Fatalf("payload: %+v immediate: %+v", payload, immediate)
}
@@ -51,7 +61,7 @@ func TestValidationContextCarriesAdjacentEvidenceAndRelevantDefinitions(t *testi
req.WorkingTranscript.Segments = req.WorkingTranscript.Segments[1:2]
req.WorkingTranscript.Segments[0].Speaker = ""
req.Glossary = nil
isolated, _ := buildValidationItems(req)
isolated, _ := buildContextItems(req)
if isolated[0].Speaker != "" || len(isolated[0].NeighboringSegments) != 0 || len(isolated[0].GlossaryEntries) != 0 {
t.Fatalf("unexpected external context: %+v", isolated)
}
@@ -73,7 +83,7 @@ func TestValidationContextFitsBudgetWithoutDiscardingCandidate(t *testing.T) {
enriched := base
enriched.NeighboringSegments = []NeighboringSegment{{Position: "following", Text: strings.Repeat("long context ", 300)}}
enriched.GlossaryEntries = []glossary.Entry{{Name: "cat", Description: strings.Repeat("long definition ", 300)}}
fit, err := fitValidationContext(enriched, required, estimator)
fit, err := fitValidationContext(enriched, required, estimator, nil)
if err != nil {
t.Fatal(err)
}
@@ -91,15 +101,17 @@ func TestValidationContextFitsBudgetWithoutDiscardingCandidate(t *testing.T) {
func TestValidationContextSelectsGlossaryForEachReplacement(t *testing.T) {
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(7, "gestures", "Jesters"), mk(11, "gestures", "Actors")})
req.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Those gestures are back."}}}
req.Glossary = []glossary.Entry{{ID: "actors", Name: "Actors"}, {ID: "jesters", Name: "Jesters"}}
payload, immediate := buildValidationItems(req)
req.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{
{ID: 1, Text: "Those gestures are back."}, {ID: 2, Text: "They met Lyra."},
}}
req.Glossary = []glossary.Entry{{ID: "lyra", Name: "Lyra"}, {ID: "actors", Name: "Actors"}, {ID: "jesters", Name: "Jesters"}}
payload, immediate := buildContextItems(req)
if len(immediate) != 0 || len(payload) != 2 {
t.Fatalf("payload: %+v immediate: %+v", payload, immediate)
}
for i, name := range []string{"Jesters", "Actors"} {
entries := payload[i].GlossaryEntries
if len(entries) != 1 || entries[0].Name != name {
if len(entries) != 2 || entries[0].Name != "Lyra" || entries[1].Name != name {
t.Fatalf("candidate %d context: %+v", i, entries)
}
}

View File

@@ -22,6 +22,7 @@ type StructuredLLMClient = structuredoutput.Client
// LLMValidationItem retains the internal proposal index until prompt encoding.
// BuildValidationVariables substitutes a one-based reference for each batch.
type LLMValidationItem struct {
segmentPosition int
CorrectionIndex int `json:"correction_index"`
Speaker string `json:"speaker"`
OriginalText string `json:"original_text"`

View File

@@ -60,6 +60,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
if req.LLMClient == nil {
return Result{}, fmt.Errorf("LLM-backed validator %q requires a structured LLM client", v.name)
}
req.Section = req.sectionLookup()
items, immediate := buildValidationItems(req)
if len(items) == 0 {
all := append([]Decision(nil), immediate...)
@@ -75,7 +76,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
maxTokens = req.Settings.Validation.MaxPromptTokens
}
batches, oversized, err := prepareValidationBatches(items, maxTokens, v.estimator)
batches, oversized, err := prepareValidationBatches(items, maxTokens, v.estimator, validationContextFor(req))
operations := make([]outcomes.Operation, 0)
llmDecisions := make([]Decision, 0)
partialResult := func() Result {
@@ -253,13 +254,6 @@ func buildValidationItems(req Request) ([]LLMValidationItem, []Decision) {
immediate := make([]Decision, 0)
req.Section = req.sectionLookup()
segments := req.Section.segments
var glossary glossaryContext
if req.GlossaryView != nil {
glossary = req.GlossaryView.context
} else {
glossary = prepareGlossaryContext(req.Glossary)
}
for _, p := range req.CandidateProposal {
position, ok := req.Section.Position(p.TargetSegmentID)
if !ok {
@@ -274,8 +268,8 @@ func buildValidationItems(req Request) ([]LLMValidationItem, []Decision) {
continue
}
neighbors, glossaryEntries := glossary.validationContext(segments, position, preview.CorrectedSegmentText)
items = append(items, LLMValidationItem{
segmentPosition: position,
CorrectionIndex: p.ProposalIndex,
Speaker: seg.Speaker,
OriginalText: p.OriginalText,
@@ -283,8 +277,6 @@ func buildValidationItems(req Request) ([]LLMValidationItem, []Decision) {
OriginalSegmentText: seg.Text,
CorrectedSegmentText: preview.CorrectedSegmentText,
Categories: append([]string(nil), seg.Categories...),
NeighboringSegments: neighbors,
GlossaryEntries: glossaryEntries,
})
}

View File

@@ -133,7 +133,8 @@ func TestValidationContextIncludesEachSharedDefinition(t *testing.T) {
{Collection: "items", ID: "bow", Name: "Bow", Category: "item"},
}
segments := []schema.Segment{{ID: 1, Text: "Shield Shield"}}
_, selected := prepareGlossaryContext(entries).validationContext(segments, 0, "Sword")
context := validationContextFor(Request{WorkingTranscript: &schema.Transcript{Segments: segments}, Glossary: entries})
_, selected := context(LLMValidationItem{CorrectedSegmentText: "Sword"})
if len(selected) != 3 || selected[0].ID != "sword" || selected[1].Category != "spell" || selected[2].ID != "shield" || selected[2].Collection != "items" {
t.Fatalf("selected=%+v", selected)
}