From d26991f0bf4528bce7b1f33759ad498a23ec127c Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Tue, 17 Nov 2026 04:25:35 +0000 Subject: [PATCH] Collect optional validation context only when it can fit --- docs/roadmap/remediation.md | 5 ++ internal/framework/validators/llm_batching.go | 4 +- internal/framework/validators/llm_budget.go | 21 ++++-- .../framework/validators/llm_budget_test.go | 36 +++++++-- internal/framework/validators/llm_context.go | 74 ++++++++++++------- .../framework/validators/llm_context_test.go | 26 +++++-- internal/framework/validators/llm_models.go | 1 + .../framework/validators/llm_validators.go | 14 +--- .../framework/validators/vocabulary_test.go | 3 +- 9 files changed, 128 insertions(+), 56 deletions(-) diff --git a/docs/roadmap/remediation.md b/docs/roadmap/remediation.md index 899d5eb..5cade85 100644 --- a/docs/roadmap/remediation.md +++ b/docs/roadmap/remediation.md @@ -331,6 +331,11 @@ form. Retain existing handling for a mismatched repair reference. ### R10 — Fit Required Validation Data Before Collecting Optional Context (P3) +**Complete.** Fitting checks required data before loading glossary and +neighbor context. Context selection reuses source matches within a validator +request and matches each revision independently. Tests verify skipped loading, +candidate-specific context, and equivalence with the serialized budget oracle. + **Affected plan:** Stages 8 and 10. **Evidence:** [Item construction](../../internal/framework/validators/llm_validators.go#L251) diff --git a/internal/framework/validators/llm_batching.go b/internal/framework/validators/llm_batching.go index 33681ac..9d51de8 100644 --- a/internal/framework/validators/llm_batching.go +++ b/internal/framework/validators/llm_batching.go @@ -14,7 +14,7 @@ type LLMValidationBatch struct { TokenCount int `json:"token_count"` } -func prepareValidationBatches(items []LLMValidationItem, budget int, estimator chunking.TokenEstimator) ([]LLMValidationBatch, []Decision, error) { +func prepareValidationBatches(items []LLMValidationItem, budget int, estimator chunking.TokenEstimator, context optionalValidationContext) ([]LLMValidationBatch, []Decision, error) { if estimator == nil { estimator = chunking.NewSimpleTokenEstimator() } @@ -24,7 +24,7 @@ func prepareValidationBatches(items []LLMValidationItem, budget int, estimator c var sizes []int var oversized []Decision for _, item := range items { - fitted, err := fitValidationContext(item, budget, estimator) + fitted, err := fitValidationContext(item, budget, estimator, context) if err != nil { return nil, oversized, err } diff --git a/internal/framework/validators/llm_budget.go b/internal/framework/validators/llm_budget.go index 3134340..3e5fb2c 100644 --- a/internal/framework/validators/llm_budget.go +++ b/internal/framework/validators/llm_budget.go @@ -3,6 +3,7 @@ package validators import ( "encoding/json" "fmt" + "slices" "gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking" ) @@ -47,21 +48,31 @@ type fittedValidationItem struct { // fitValidationContext tries the required payload followed by glossary entries // and then neighboring segments. The first excess ends the usable prefix. -func fitValidationContext(item LLMValidationItem, budget int, estimator chunking.TokenEstimator) (fittedValidationItem, error) { +func fitValidationContext(item LLMValidationItem, budget int, estimator chunking.TokenEstimator, context optionalValidationContext) (result fittedValidationItem, err error) { fitted := fittedValidationItem{item: item} - fitted.item.GlossaryEntries = item.GlossaryEntries[:0] - fitted.item.NeighboringSegments = item.NeighboringSegments[:0] + fitted.item.GlossaryEntries = nil + fitted.item.NeighboringSegments = nil simple, _ := estimator.(*chunking.SimpleTokenEstimator) - var err error if simple != nil { fitted.cost, err = measureValidationItem(simple, fitted.item) fitted.tokens = fitted.cost.arrayTokens() } else { fitted.tokens, err = estimateBatchTokens(estimator, []LLMValidationItem{fitted.item}) } - if err != nil || fitted.tokens > budget { + if err != nil || fitted.tokens > budget || (simple != nil && fitted.tokens == budget) { return fitted, err } + if context != nil { + item.NeighboringSegments, item.GlossaryEntries = context(item) + } + defer func() { + if len(result.item.GlossaryEntries) < len(item.GlossaryEntries) { + result.item.GlossaryEntries = slices.Clone(result.item.GlossaryEntries) + } + if len(result.item.NeighboringSegments) < len(item.NeighboringSegments) { + result.item.NeighboringSegments = slices.Clone(result.item.NeighboringSegments) + } + }() try := func(candidate LLMValidationItem, field string, first bool, entry any) (bool, error) { cost := fitted.cost var tokens int diff --git a/internal/framework/validators/llm_budget_test.go b/internal/framework/validators/llm_budget_test.go index 387fe9b..8d4ab63 100644 --- a/internal/framework/validators/llm_budget_test.go +++ b/internal/framework/validators/llm_budget_test.go @@ -1,8 +1,8 @@ package validators import ( + "bytes" "encoding/json" - "reflect" "slices" "strings" "testing" @@ -97,7 +97,7 @@ func TestComposedValidationBudgetMatchesSerializedPayload(t *testing.T) { simple := chunking.NewSimpleTokenEstimator() full := serializedTokenCount(t, items) for budget := 1; budget <= full; budget++ { - got, rejected, err := prepareValidationBatches(items, budget, simple) + got, rejected, err := prepareValidationBatches(items, budget, simple, nil) want, wantRejected := serializedBudgetOracle(t, items, budget) var rejectedIndices []int for _, decision := range rejected { @@ -106,7 +106,9 @@ func TestComposedValidationBudgetMatchesSerializedPayload(t *testing.T) { } rejectedIndices = append(rejectedIndices, decision.ProposalIndex) } - if err != nil || !reflect.DeepEqual(got, want) || !slices.Equal(rejectedIndices, wantRejected) { + gotJSON, gotErr := json.Marshal(got) + wantJSON, wantErr := json.Marshal(want) + if err != nil || gotErr != nil || wantErr != nil || !bytes.Equal(gotJSON, wantJSON) || !slices.Equal(rejectedIndices, wantRejected) { t.Fatalf("budget %d: composed=%+v rejected=%+v err=%v; serialized=%+v rejected=%+v", budget, got, rejected, err, want, wantRejected) } @@ -153,7 +155,7 @@ func TestValidationFitsRequiredDataBeforeScanningOptionalContext(t *testing.T) { item.GlossaryEntries = append([]glossary.Entry{first, {Description: strings.Repeat("large body ", 500)}}, make([]glossary.Entry, 5000)...) item.NeighboringSegments = []NeighboringSegment{{Text: "neighbor"}} estimator := &countingEstimator{} - fitted, err := fitValidationContext(item, budget, estimator) + fitted, err := fitValidationContext(item, budget, estimator, nil) if err != nil || len(fitted.item.GlossaryEntries) != 1 || fitted.item.GlossaryEntries[0].ID != first.ID || len(fitted.item.NeighboringSegments) != 0 || fitted.tokens != budget { t.Fatalf("incorrect usable prefix: %+v (%v)", fitted, err) @@ -162,8 +164,32 @@ func TestValidationFitsRequiredDataBeforeScanningOptionalContext(t *testing.T) { t.Fatalf("measured beyond the first excess: %d measurements", estimator.measurements) } estimator.measurements = 0 - fitted, err = fitValidationContext(item, 1, estimator) + fitted, err = fitValidationContext(item, 1, estimator, nil) if err != nil || fitted.tokens <= 1 || estimator.measurements != 1 { t.Fatalf("oversized required data: tokens=%d measurements=%d error=%v", fitted.tokens, estimator.measurements, err) } } + +func TestValidationSkipsOptionalContextWithoutBudget(t *testing.T) { + item := LLMValidationItem{CorrectionIndex: 7, OriginalText: "teh", CorrectedText: "the"} + estimator := chunking.NewSimpleTokenEstimator() + required := serializedTokenCount(t, []LLMValidationItem{item}) + loads := 0 + optional := func(LLMValidationItem) ([]NeighboringSegment, []glossary.Entry) { + loads++ + return nil, []glossary.Entry{{Name: "optional"}} + } + for _, budget := range []int{required - 1, required} { + batches, oversized, err := prepareValidationBatches([]LLMValidationItem{item}, budget, estimator, optional) + if err != nil || loads != 0 { + t.Fatalf("budget=%d loads=%d error=%v", budget, loads, err) + } + if budget < required { + if len(batches) != 0 || len(oversized) != 1 { + t.Fatalf("oversized result: batches=%+v rejected=%+v", batches, oversized) + } + } else if len(batches) != 1 || len(oversized) != 0 || batches[0].TokenCount != required { + t.Fatalf("exact fit: batches=%+v rejected=%+v", batches, oversized) + } + } +} diff --git a/internal/framework/validators/llm_context.go b/internal/framework/validators/llm_context.go index c9249b7..924742f 100644 --- a/internal/framework/validators/llm_context.go +++ b/internal/framework/validators/llm_context.go @@ -33,34 +33,58 @@ func prepareGlossaryContext(entries []glossary.Entry) glossaryContext { return glossaryContext{entries: entries, positions: positions, vocabulary: glossary.NewProtectedVocabulary(glossary.Vocabulary(entries, nil))} } -// validationContext supplies adjacent source segments within the current chunk -// and glossary definitions mentioned in the candidate or that local context. -// These fields travel in each item so batching accounts for their size. -func (view glossaryContext) validationContext(segments []schema.Segment, position int, corrected string) ([]NeighboringSegment, []glossary.Entry) { - var neighbors []NeighboringSegment - source := segments[position].Text - if position > 0 { - neighbors = append(neighbors, neighboringSegment("preceding", segments[position-1])) - } - if position+1 < len(segments) { - neighbors = append(neighbors, neighboringSegment("following", segments[position+1])) - } +type optionalValidationContext func(LLMValidationItem) ([]NeighboringSegment, []glossary.Entry) - texts := []string{source, corrected} - for _, neighbor := range neighbors { - texts = append(texts, neighbor.Text) +// validationContextFor builds optional context on demand. Unchanged segment +// matches are shared within the request. Corrected text is candidate-specific. +func validationContextFor(req Request) optionalValidationContext { + segments := req.sectionLookup().segments + var view *glossaryContext + if req.GlossaryView != nil { + view = &req.GlossaryView.context } - mentioned := view.vocabulary.MentionedIdentities(texts...) - var matched []int - for identity := range mentioned { - matched = append(matched, view.positions[identity]...) + var sourceMentions map[int]map[glossary.Identity]struct{} + return func(item LLMValidationItem) ([]NeighboringSegment, []glossary.Entry) { + position := item.segmentPosition + var neighbors []NeighboringSegment + if position > 0 { + neighbors = append(neighbors, neighboringSegment("preceding", segments[position-1])) + } + if position+1 < len(segments) { + neighbors = append(neighbors, neighboringSegment("following", segments[position+1])) + } + if view == nil { + prepared := prepareGlossaryContext(req.Glossary) + view = &prepared + } + if view.vocabulary.Empty() { + return neighbors, nil + } + if sourceMentions == nil { + sourceMentions = make(map[int]map[glossary.Identity]struct{}) + } + mentioned := view.vocabulary.MentionedIdentities(item.CorrectedSegmentText) + for i := max(0, position-1); i < min(len(segments), position+2); i++ { + identities, ok := sourceMentions[i] + if !ok { + identities = view.vocabulary.MentionedIdentities(segments[i].Text) + sourceMentions[i] = identities + } + for id := range identities { + mentioned[id] = struct{}{} + } + } + var matched []int + for identity := range mentioned { + matched = append(matched, view.positions[identity]...) + } + slices.Sort(matched) + var entries []glossary.Entry + for _, index := range matched { + entries = append(entries, view.entries[index]) + } + return neighbors, entries } - slices.Sort(matched) - var entries []glossary.Entry - for _, index := range matched { - entries = append(entries, view.entries[index]) - } - return neighbors, entries } func neighboringSegment(position string, segment schema.Segment) NeighboringSegment { diff --git a/internal/framework/validators/llm_context_test.go b/internal/framework/validators/llm_context_test.go index 75d48a2..5c767c4 100644 --- a/internal/framework/validators/llm_context_test.go +++ b/internal/framework/validators/llm_context_test.go @@ -11,6 +11,16 @@ import ( "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" ) +func buildContextItems(req Request) ([]LLMValidationItem, []Decision) { + req.Section = req.sectionLookup() + items, immediate := buildValidationItems(req) + context := validationContextFor(req) + for i := range items { + items[i].NeighboringSegments, items[i].GlossaryEntries = context(items[i]) + } + return items, immediate +} + func TestValidationContextCarriesAdjacentEvidenceAndRelevantDefinitions(t *testing.T) { req := makeReq([]proposals.EnrichedCorrectionProposal{mk(7, "gestures", "Jesters")}) req.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{ @@ -25,7 +35,7 @@ func TestValidationContextCarriesAdjacentEvidenceAndRelevantDefinitions(t *testi {ID: "lyra", Name: "Lyra", Description: "The host"}, {ID: "unrelated", Name: "Unrelated", Description: "Not mentioned"}, } - payload, immediate := buildValidationItems(req) + payload, immediate := buildContextItems(req) if len(immediate) != 0 || len(payload) != 1 { t.Fatalf("payload: %+v immediate: %+v", payload, immediate) } @@ -51,7 +61,7 @@ func TestValidationContextCarriesAdjacentEvidenceAndRelevantDefinitions(t *testi req.WorkingTranscript.Segments = req.WorkingTranscript.Segments[1:2] req.WorkingTranscript.Segments[0].Speaker = "" req.Glossary = nil - isolated, _ := buildValidationItems(req) + isolated, _ := buildContextItems(req) if isolated[0].Speaker != "" || len(isolated[0].NeighboringSegments) != 0 || len(isolated[0].GlossaryEntries) != 0 { t.Fatalf("unexpected external context: %+v", isolated) } @@ -73,7 +83,7 @@ func TestValidationContextFitsBudgetWithoutDiscardingCandidate(t *testing.T) { enriched := base enriched.NeighboringSegments = []NeighboringSegment{{Position: "following", Text: strings.Repeat("long context ", 300)}} enriched.GlossaryEntries = []glossary.Entry{{Name: "cat", Description: strings.Repeat("long definition ", 300)}} - fit, err := fitValidationContext(enriched, required, estimator) + fit, err := fitValidationContext(enriched, required, estimator, nil) if err != nil { t.Fatal(err) } @@ -91,15 +101,17 @@ func TestValidationContextFitsBudgetWithoutDiscardingCandidate(t *testing.T) { func TestValidationContextSelectsGlossaryForEachReplacement(t *testing.T) { req := makeReq([]proposals.EnrichedCorrectionProposal{mk(7, "gestures", "Jesters"), mk(11, "gestures", "Actors")}) - req.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Those gestures are back."}}} - req.Glossary = []glossary.Entry{{ID: "actors", Name: "Actors"}, {ID: "jesters", Name: "Jesters"}} - payload, immediate := buildValidationItems(req) + req.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{ + {ID: 1, Text: "Those gestures are back."}, {ID: 2, Text: "They met Lyra."}, + }} + req.Glossary = []glossary.Entry{{ID: "lyra", Name: "Lyra"}, {ID: "actors", Name: "Actors"}, {ID: "jesters", Name: "Jesters"}} + payload, immediate := buildContextItems(req) if len(immediate) != 0 || len(payload) != 2 { t.Fatalf("payload: %+v immediate: %+v", payload, immediate) } for i, name := range []string{"Jesters", "Actors"} { entries := payload[i].GlossaryEntries - if len(entries) != 1 || entries[0].Name != name { + if len(entries) != 2 || entries[0].Name != "Lyra" || entries[1].Name != name { t.Fatalf("candidate %d context: %+v", i, entries) } } diff --git a/internal/framework/validators/llm_models.go b/internal/framework/validators/llm_models.go index 91ae62a..7b8cba8 100644 --- a/internal/framework/validators/llm_models.go +++ b/internal/framework/validators/llm_models.go @@ -22,6 +22,7 @@ type StructuredLLMClient = structuredoutput.Client // LLMValidationItem retains the internal proposal index until prompt encoding. // BuildValidationVariables substitutes a one-based reference for each batch. type LLMValidationItem struct { + segmentPosition int CorrectionIndex int `json:"correction_index"` Speaker string `json:"speaker"` OriginalText string `json:"original_text"` diff --git a/internal/framework/validators/llm_validators.go b/internal/framework/validators/llm_validators.go index fe61444..9ee6b64 100644 --- a/internal/framework/validators/llm_validators.go +++ b/internal/framework/validators/llm_validators.go @@ -60,6 +60,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, if req.LLMClient == nil { return Result{}, fmt.Errorf("LLM-backed validator %q requires a structured LLM client", v.name) } + req.Section = req.sectionLookup() items, immediate := buildValidationItems(req) if len(items) == 0 { all := append([]Decision(nil), immediate...) @@ -75,7 +76,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, maxTokens = req.Settings.Validation.MaxPromptTokens } - batches, oversized, err := prepareValidationBatches(items, maxTokens, v.estimator) + batches, oversized, err := prepareValidationBatches(items, maxTokens, v.estimator, validationContextFor(req)) operations := make([]outcomes.Operation, 0) llmDecisions := make([]Decision, 0) partialResult := func() Result { @@ -253,13 +254,6 @@ func buildValidationItems(req Request) ([]LLMValidationItem, []Decision) { immediate := make([]Decision, 0) req.Section = req.sectionLookup() segments := req.Section.segments - var glossary glossaryContext - if req.GlossaryView != nil { - glossary = req.GlossaryView.context - } else { - glossary = prepareGlossaryContext(req.Glossary) - } - for _, p := range req.CandidateProposal { position, ok := req.Section.Position(p.TargetSegmentID) if !ok { @@ -274,8 +268,8 @@ func buildValidationItems(req Request) ([]LLMValidationItem, []Decision) { continue } - neighbors, glossaryEntries := glossary.validationContext(segments, position, preview.CorrectedSegmentText) items = append(items, LLMValidationItem{ + segmentPosition: position, CorrectionIndex: p.ProposalIndex, Speaker: seg.Speaker, OriginalText: p.OriginalText, @@ -283,8 +277,6 @@ func buildValidationItems(req Request) ([]LLMValidationItem, []Decision) { OriginalSegmentText: seg.Text, CorrectedSegmentText: preview.CorrectedSegmentText, Categories: append([]string(nil), seg.Categories...), - NeighboringSegments: neighbors, - GlossaryEntries: glossaryEntries, }) } diff --git a/internal/framework/validators/vocabulary_test.go b/internal/framework/validators/vocabulary_test.go index 30e5b94..aee4f61 100644 --- a/internal/framework/validators/vocabulary_test.go +++ b/internal/framework/validators/vocabulary_test.go @@ -133,7 +133,8 @@ func TestValidationContextIncludesEachSharedDefinition(t *testing.T) { {Collection: "items", ID: "bow", Name: "Bow", Category: "item"}, } segments := []schema.Segment{{ID: 1, Text: "Shield Shield"}} - _, selected := prepareGlossaryContext(entries).validationContext(segments, 0, "Sword") + context := validationContextFor(Request{WorkingTranscript: &schema.Transcript{Segments: segments}, Glossary: entries}) + _, selected := context(LLMValidationItem{CorrectedSegmentText: "Sword"}) if len(selected) != 3 || selected[0].ID != "sword" || selected[1].Category != "spell" || selected[2].ID != "shield" || selected[2].Collection != "items" { t.Fatalf("selected=%+v", selected) }