Validate internal decisions and reconcile exact model references
This commit is contained in:
@@ -2,6 +2,7 @@ package validators
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
@@ -9,6 +10,7 @@ import (
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/jsonnumber"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/stagename"
|
||||
@@ -58,8 +60,8 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
if req.LLMClient == nil {
|
||||
return Result{}, fmt.Errorf("LLM-backed validator %q requires a structured LLM client", v.name)
|
||||
}
|
||||
validationReq, immediate := BuildLLMValidationRequest(v.name, v.validatorType, req)
|
||||
if len(validationReq.Items) == 0 {
|
||||
items, immediate := buildValidationItems(req)
|
||||
if len(items) == 0 {
|
||||
all := append([]Decision(nil), immediate...)
|
||||
sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex })
|
||||
if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil {
|
||||
@@ -73,7 +75,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
maxTokens = req.Settings.Validation.MaxPromptTokens
|
||||
}
|
||||
|
||||
batches, oversized, err := prepareValidationBatches(validationReq.Items, maxTokens, v.estimator)
|
||||
batches, oversized, err := prepareValidationBatches(items, maxTokens, v.estimator)
|
||||
operations := make([]outcomes.Operation, 0)
|
||||
llmDecisions := make([]Decision, 0)
|
||||
partialResult := func() Result {
|
||||
@@ -239,7 +241,7 @@ func errPayload(err error) any {
|
||||
return map[string]any{"error": err.Error()}
|
||||
}
|
||||
|
||||
func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorType, req Request) (LLMValidationRequest, []Decision) {
|
||||
func buildValidationItems(req Request) ([]LLMValidationItem, []Decision) {
|
||||
items := make([]LLMValidationItem, 0, len(req.CandidateProposal))
|
||||
immediate := make([]Decision, 0)
|
||||
req.Section = req.sectionLookup()
|
||||
@@ -268,7 +270,6 @@ func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorT
|
||||
neighbors, glossaryEntries := glossary.validationContext(segments, position, preview.CorrectedSegmentText)
|
||||
items = append(items, LLMValidationItem{
|
||||
CorrectionIndex: p.ProposalIndex,
|
||||
SegmentID: p.TargetSegmentID,
|
||||
Speaker: seg.Speaker,
|
||||
OriginalText: p.OriginalText,
|
||||
CorrectedText: p.CorrectedText,
|
||||
@@ -280,31 +281,25 @@ func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorT
|
||||
})
|
||||
}
|
||||
|
||||
return LLMValidationRequest{
|
||||
ValidatorName: validatorName,
|
||||
ValidatorType: validatorType,
|
||||
ModuleKey: req.ModuleKey,
|
||||
ModuleInstance: req.ModuleInstance,
|
||||
ReplacementPolicy: string(req.ReplacementPolicy),
|
||||
Items: items,
|
||||
}, immediate
|
||||
return items, immediate
|
||||
}
|
||||
|
||||
type validationReconciliation struct {
|
||||
Decisions []Decision `json:"decisions"`
|
||||
UnavailableIndices []int `json:"unavailable_indices,omitempty"`
|
||||
IgnoredIndices []int `json:"ignored_indices,omitempty"`
|
||||
Decisions []Decision `json:"decisions"`
|
||||
UnavailableIndices []int `json:"unavailable_indices,omitempty"`
|
||||
IgnoredIndices []json.Number `json:"ignored_indices,omitempty"`
|
||||
}
|
||||
|
||||
func reconcileLLMResponse(items []LLMValidationItem, response LLMValidationResponse) validationReconciliation {
|
||||
result := validationReconciliation{Decisions: make([]Decision, 0, len(items))}
|
||||
seen := make(map[int][]LLMValidationDecision, len(items))
|
||||
for _, d := range response.Validations {
|
||||
if d.CorrectionIndex < 1 || d.CorrectionIndex > len(items) {
|
||||
index, ok := jsonnumber.PositiveIndex(d.CorrectionIndex.String(), len(items))
|
||||
if !ok {
|
||||
result.IgnoredIndices = append(result.IgnoredIndices, d.CorrectionIndex)
|
||||
continue
|
||||
}
|
||||
seen[d.CorrectionIndex] = append(seen[d.CorrectionIndex], d)
|
||||
seen[index] = append(seen[index], d)
|
||||
}
|
||||
|
||||
for i, item := range items {
|
||||
|
||||
Reference in New Issue
Block a user