eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Reuse validation costs when fitting and batching

Browse Source
This commit is contained in:
Eric Rakestraw
2026-11-10 20:45:38 +00:00
parent c16d61e091
commit a9b7c37a7e
7 changed files with 267 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
internal/framework/validators/llm_validators.go
View File

@@ -75,10 +75,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
maxTokens = req.Settings.Validation.MaxPromptTokens
}
for i := range validationReq.Items {
validationReq.Items[i] = fitValidationContext(validationReq.Items[i], maxTokens, v.estimator)
}
oversized := oversizedValidationDecisions(validationReq.Items, maxTokens, v.estimator)
batches, oversized, err := prepareValidationBatches(validationReq.Items, maxTokens, v.estimator)
operations := make([]outcomes.Operation, 0)
llmDecisions := make([]Decision, 0)
partialResult := func() Result {
@@ -110,18 +107,6 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
ProposalIndices: indices,
})
}
itemsForBatching := filterItemsByDecision(validationReq.Items, oversized)
if len(itemsForBatching) == 0 {
result := partialResult()
all := result.Decisions
if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil {
return result, err
}
sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex })
return result, nil
}
batches, err := ChunkLLMValidationItems(itemsForBatching, maxTokens, v.estimator)
if err != nil {
return partialResult(), err
}
@@ -355,17 +340,21 @@ func reconcileLLMResponse(items []LLMValidationItem, response LLMValidationRespo
}
func oversizedValidationDecisions(items []LLMValidationItem, maxPromptTokens int, estimator chunking.TokenEstimator) []Decision {
out := make([]Decision, 0)
var out []Decision
for _, item := range items {
singleTokens, err := estimateBatchTokens(estimator, []LLMValidationItem{item})
if err != nil || singleTokens <= maxPromptTokens {
continue
}
out = append(out, unavailable(item.CorrectionIndex, ReasonValidatorInputTooLarge, "validation input exceeds max prompt tokens"))
out = append(out, oversizedValidationDecision(item.CorrectionIndex))
}
return out
}
func oversizedValidationDecision(index int) Decision {
return unavailable(index, ReasonValidatorInputTooLarge, "validation input exceeds max prompt tokens")
}
func filterItemsByDecision(items []LLMValidationItem, decisions []Decision) []LLMValidationItem {
if len(decisions) == 0 {
return append([]LLMValidationItem(nil), items...)