Validate internal decisions and reconcile exact model references
This commit is contained in:
@@ -37,6 +37,7 @@ func (v OriginalTextPresenceValidator) Name() string { return "original_text_pre
|
||||
|
||||
func (v OriginalTextPresenceValidator) Validate(_ context.Context, req Request) (Result, error) {
|
||||
section := req.sectionLookup()
|
||||
req.Section = section
|
||||
|
||||
decisions := make([]Decision, 0, len(req.CandidateProposal))
|
||||
for _, c := range req.CandidateProposal {
|
||||
@@ -49,6 +50,10 @@ func (v OriginalTextPresenceValidator) Validate(_ context.Context, req Request)
|
||||
decisions = append(decisions, rejection(c.ProposalIndex, ReasonMissingOriginalText, "original_text was not found in current segment text"))
|
||||
continue
|
||||
}
|
||||
if req.IndependentEvidence && req.Preview(c).SkipReason == proposals.SkipReasonAmbiguousOriginal {
|
||||
decisions = append(decisions, rejection(c.ProposalIndex, string(proposals.SkipReasonAmbiguousOriginal), "original_text matches multiple locations in the target segment"))
|
||||
continue
|
||||
}
|
||||
decisions = append(decisions, approval(c.ProposalIndex))
|
||||
}
|
||||
if err := EnforceDecisionCardinality(req.CandidateProposal, decisions); err != nil {
|
||||
|
||||
@@ -25,15 +25,15 @@ func TestValidationContextCarriesAdjacentEvidenceAndRelevantDefinitions(t *testi
|
||||
{ID: "lyra", Name: "Lyra", Description: "The host"},
|
||||
{ID: "unrelated", Name: "Unrelated", Description: "Not mentioned"},
|
||||
}
|
||||
payload, immediate := BuildLLMValidationRequest("spoken_form_plausibility", LLMValidatorTypeSpokenFormPlausibility, req)
|
||||
if len(immediate) != 0 || len(payload.Items) != 1 {
|
||||
payload, immediate := buildValidationItems(req)
|
||||
if len(immediate) != 0 || len(payload) != 1 {
|
||||
t.Fatalf("payload: %+v immediate: %+v", payload, immediate)
|
||||
}
|
||||
item := payload.Items[0]
|
||||
item := payload[0]
|
||||
if len(item.NeighboringSegments) != 2 || item.NeighboringSegments[0].Position != "preceding" || item.NeighboringSegments[1].Position != "following" || len(item.GlossaryEntries) != 2 || item.GlossaryEntries[0].Name != "Jesters" || item.GlossaryEntries[1].Name != "Lyra" {
|
||||
t.Fatalf("context: %+v", item)
|
||||
}
|
||||
variables, err := BuildValidationVariables(payload.Items, "")
|
||||
variables, err := BuildValidationVariables(payload, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -41,7 +41,7 @@ func TestValidationContextCarriesAdjacentEvidenceAndRelevantDefinitions(t *testi
|
||||
if err := json.Unmarshal([]byte(variables["PayloadJSON"]), &rendered); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(variables["PayloadJSON"], `"segment_id":`) || rendered[0].CorrectionIndex != 1 || payload.Items[0].CorrectionIndex != 7 {
|
||||
if strings.Contains(variables["PayloadJSON"], `"segment_id":`) || rendered[0].CorrectionIndex != 1 || payload[0].CorrectionIndex != 7 {
|
||||
t.Fatalf("leaked internal identity or mutated request: %s", variables["PayloadJSON"])
|
||||
}
|
||||
if rendered[0].Speaker != "Guest" || len(rendered[0].NeighboringSegments) != 2 || rendered[0].NeighboringSegments[0].Speaker != "Host" || rendered[0].GlossaryEntries[0].Description != "A traveling troupe" {
|
||||
@@ -51,15 +51,15 @@ func TestValidationContextCarriesAdjacentEvidenceAndRelevantDefinitions(t *testi
|
||||
req.WorkingTranscript.Segments = req.WorkingTranscript.Segments[1:2]
|
||||
req.WorkingTranscript.Segments[0].Speaker = ""
|
||||
req.Glossary = nil
|
||||
isolated, _ := BuildLLMValidationRequest("editorial_review", LLMValidatorTypeEditorialReview, req)
|
||||
if isolated.Items[0].Speaker != "" || len(isolated.Items[0].NeighboringSegments) != 0 || len(isolated.Items[0].GlossaryEntries) != 0 {
|
||||
isolated, _ := buildValidationItems(req)
|
||||
if isolated[0].Speaker != "" || len(isolated[0].NeighboringSegments) != 0 || len(isolated[0].GlossaryEntries) != 0 {
|
||||
t.Fatalf("unexpected external context: %+v", isolated)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationContextFitsBudgetWithoutDiscardingCandidate(t *testing.T) {
|
||||
estimator := chunking.NewSimpleTokenEstimator()
|
||||
base := LLMValidationItem{CorrectionIndex: 7, SegmentID: 1, OriginalText: "teh", CorrectedText: "the", OriginalSegmentText: "teh cat", CorrectedSegmentText: "the cat"}
|
||||
base := LLMValidationItem{CorrectionIndex: 7, OriginalText: "teh", CorrectedText: "the", OriginalSegmentText: "teh cat", CorrectedSegmentText: "the cat"}
|
||||
required, err := estimateBatchTokens(estimator, []LLMValidationItem{base})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -89,12 +89,12 @@ func TestValidationContextSelectsGlossaryForEachReplacement(t *testing.T) {
|
||||
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(7, "gestures", "Jesters"), mk(11, "gestures", "Actors")})
|
||||
req.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Those gestures are back."}}}
|
||||
req.Glossary = []glossary.Entry{{ID: "actors", Name: "Actors"}, {ID: "jesters", Name: "Jesters"}}
|
||||
payload, immediate := BuildLLMValidationRequest("editorial_review", LLMValidatorTypeEditorialReview, req)
|
||||
if len(immediate) != 0 || len(payload.Items) != 2 {
|
||||
payload, immediate := buildValidationItems(req)
|
||||
if len(immediate) != 0 || len(payload) != 2 {
|
||||
t.Fatalf("payload: %+v immediate: %+v", payload, immediate)
|
||||
}
|
||||
for i, name := range []string{"Jesters", "Actors"} {
|
||||
entries := payload.Items[i].GlossaryEntries
|
||||
entries := payload[i].GlossaryEntries
|
||||
if len(entries) != 1 || entries[0].Name != name {
|
||||
t.Fatalf("candidate %d context: %+v", i, entries)
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||
ck "go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
@@ -50,28 +51,7 @@ func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) {
|
||||
}
|
||||
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
|
||||
})
|
||||
dir := t.TempDir()
|
||||
main := filepath.Join(dir, "config.yml")
|
||||
if err := os.WriteFile(main, []byte("version: 3\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte("profiles:\n - {id: base, endpoint: 'https://example.invalid/v1', model: local-model, api_key_required: false}\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
loaded, err := config.Load(ck.Input{Source: ck.File(main)}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runtime, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
settings := loaded.Config.LLM
|
||||
settings.TransientRetries, settings.RepairAttempts = 0, 1
|
||||
client, err := runtime.NewRoleClient("validation", "base", settings, llm.CallHooks{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client := testValidationClient(t, transport)
|
||||
validator, err := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -91,3 +71,98 @@ func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The transport supplies provider responses; PromptKit performs structured
|
||||
// validation and Audita decodes the same response through the role client.
|
||||
func testValidationClient(t *testing.T, transport http.RoundTripper) structuredoutput.Client {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
main := filepath.Join(dir, "config.yml")
|
||||
if err := os.WriteFile(main, []byte("version: 3\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte("profiles:\n - {id: base, endpoint: 'https://example.invalid/v1', model: local-model, api_key_required: false}\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
loaded, err := config.Load(ck.Input{Source: ck.File(main)}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runtime, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
settings := loaded.Config.LLM
|
||||
settings.TransientRetries, settings.RepairAttempts = 0, 1
|
||||
client, err := runtime.NewRoleClient("validation", "base", settings, llm.CallHooks{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return client
|
||||
}
|
||||
|
||||
func TestValidationDecodesExactReferencesBeforeReconciliation(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, response string
|
||||
indices []int
|
||||
want []DecisionDisposition
|
||||
reasons []string
|
||||
failure outcomes.ReasonCode
|
||||
}{
|
||||
{
|
||||
name: "usable decision beside extra",
|
||||
response: `{"validations":[{"correction_index":1.00e0,"approved":true,"reason":"accept"},{"correction_index":9223372036854775808,"approved":true,"reason":"extra"}]}`,
|
||||
indices: []int{7},
|
||||
want: []DecisionDisposition{DispositionApproved},
|
||||
reasons: []string{ReasonApproved},
|
||||
},
|
||||
{
|
||||
name: "usable siblings beside duplicate and omission",
|
||||
response: `{"validations":[{"correction_index":1e0,"approved":true,"reason":"accept"},{"correction_index":2,"approved":true,"reason":"duplicate"},{"correction_index":20e-1,"approved":false,"reason":"duplicate"},{"correction_index":4.000,"approved":false,"reason":"reject"},{"correction_index":9223372036854775808,"approved":true,"reason":"extra"}]}`,
|
||||
indices: []int{7, 11, 14, 23},
|
||||
want: []DecisionDisposition{DispositionApproved, DispositionUnavailable, DispositionUnavailable, DispositionRejected},
|
||||
reasons: []string{ReasonApproved, ReasonDuplicateValidationDecision, ReasonMissingValidationDecision, "llm_rejected"},
|
||||
failure: outcomes.ReasonValidationDecisionsInvalid,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
calls := 0
|
||||
client := testValidationClient(t, validationTransport(func(req *http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
content, _ := json.Marshal(tc.response)
|
||||
body := fmt.Sprintf(`{"choices":[{"message":{"content":%s}}]}`, content)
|
||||
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: req}, nil
|
||||
}))
|
||||
var candidates []proposals.EnrichedCorrectionProposal
|
||||
for _, index := range tc.indices {
|
||||
candidates = append(candidates, mk(index, "gestures", "Jesters"))
|
||||
}
|
||||
validator, err := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sink := &captureValidationDiagnosticsWriter{}
|
||||
req := makeReq(candidates)
|
||||
req.LLMClient = client
|
||||
req.Interactions = sink
|
||||
result, err := validator.Validate(context.Background(), req)
|
||||
if err != nil || calls != 1 || len(result.Decisions) != len(tc.want) || len(result.Operations) != 1 {
|
||||
t.Fatalf("result=%+v calls=%d error=%v", result, calls, err)
|
||||
}
|
||||
for i, decision := range result.Decisions {
|
||||
if decision.ProposalIndex != tc.indices[i] || decision.Disposition != tc.want[i] || decision.ReasonCode != tc.reasons[i] {
|
||||
t.Fatalf("decision %d: %+v", i, decision)
|
||||
}
|
||||
}
|
||||
operation := result.Operations[0]
|
||||
if operation.FailureReason != tc.failure || (tc.failure == "" && (len(operation.RecoveryReasons) != 1 || operation.RecoveryReasons[0] != outcomes.RecoveryValidationDecisionsFiltered)) {
|
||||
t.Fatalf("operation: %+v", operation)
|
||||
}
|
||||
payload := sink.lastResponsePayload.(map[string]any)
|
||||
diagnostics := payload["reconciliation"].(validationReconciliation)
|
||||
if len(diagnostics.IgnoredIndices) != 1 || diagnostics.IgnoredIndices[0] != "9223372036854775808" || payload["final_content"] != tc.response {
|
||||
t.Fatalf("reference evidence changed: %+v", payload)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package validators
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||
)
|
||||
@@ -21,7 +23,6 @@ type StructuredLLMClient = structuredoutput.Client
|
||||
// BuildValidationVariables substitutes a one-based reference for each batch.
|
||||
type LLMValidationItem struct {
|
||||
CorrectionIndex int `json:"correction_index"`
|
||||
SegmentID int `json:"-"`
|
||||
Speaker string `json:"speaker"`
|
||||
OriginalText string `json:"original_text"`
|
||||
CorrectedText string `json:"corrected_text"`
|
||||
@@ -32,21 +33,11 @@ type LLMValidationItem struct {
|
||||
GlossaryEntries []glossary.Entry `json:"glossary_entries,omitempty"`
|
||||
}
|
||||
|
||||
// LLMValidationRequest is the canonical request model for LLM-backed validators.
|
||||
type LLMValidationRequest struct {
|
||||
ValidatorName string `json:"validator_name"`
|
||||
ValidatorType LLMValidatorType `json:"validator_type"`
|
||||
ModuleKey string `json:"module_key"`
|
||||
ModuleInstance string `json:"module_instance"`
|
||||
ReplacementPolicy string `json:"replacement_policy"`
|
||||
Items []LLMValidationItem `json:"corrections"`
|
||||
}
|
||||
|
||||
// LLMValidationDecision contains a request-local correction_index, not a proposal index.
|
||||
type LLMValidationDecision struct {
|
||||
CorrectionIndex int `json:"correction_index"`
|
||||
Approved bool `json:"approved"`
|
||||
Reason string `json:"reason"`
|
||||
CorrectionIndex json.Number `json:"correction_index"`
|
||||
Approved bool `json:"approved"`
|
||||
Reason string `json:"reason"`
|
||||
}
|
||||
|
||||
type LLMValidationResponse struct {
|
||||
|
||||
@@ -48,9 +48,9 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
|
||||
}
|
||||
failed := &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonTransportFailed, Message: "model transport failed"}
|
||||
client := &scriptedValidationClient{replies: []validationReply{
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}},
|
||||
{completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 1}}, err: failed},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}},
|
||||
}}
|
||||
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
|
||||
validator.SetTokenEstimator(perItemEstimator{})
|
||||
@@ -73,8 +73,8 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
|
||||
|
||||
survivors := []proposals.EnrichedCorrectionProposal{candidates[0], candidates[2]}
|
||||
nextClient := &scriptedValidationClient{replies: []validationReply{
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}},
|
||||
}}
|
||||
nextValidator, _ := NewLLMBackedValidator("meaning_reversal_review", LLMValidatorTypeMeaningReversal)
|
||||
nextValidator.SetTokenEstimator(perItemEstimator{})
|
||||
@@ -101,7 +101,7 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
|
||||
|
||||
func TestValidationDomainInvalidAfterRepairIsAnError(t *testing.T) {
|
||||
client := &scriptedValidationClient{replies: []validationReply{{
|
||||
response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Reason: "ok"}}},
|
||||
response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "99", Approved: true, Reason: "ok"}}},
|
||||
completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 2, StructuralRepairs: 1}},
|
||||
}}}
|
||||
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
|
||||
@@ -116,8 +116,8 @@ func TestValidationDomainInvalidAfterRepairIsAnError(t *testing.T) {
|
||||
func TestValidationExtraDecisionsMergeActualRecoveries(t *testing.T) {
|
||||
client := &scriptedValidationClient{replies: []validationReply{{
|
||||
response: LLMValidationResponse{Validations: []LLMValidationDecision{
|
||||
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
|
||||
{CorrectionIndex: 99, Approved: true, Reason: "extra"},
|
||||
{CorrectionIndex: "1", Approved: true, Reason: "ok"},
|
||||
{CorrectionIndex: "99", Approved: true, Reason: "extra"},
|
||||
}},
|
||||
completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 3, TransientRetries: 1, StructuralRepairs: 1}},
|
||||
}}}
|
||||
@@ -137,9 +137,9 @@ func TestValidationExtraDecisionsMergeActualRecoveries(t *testing.T) {
|
||||
func TestValidationFatalFailureKeepsEarlierFactsAndStops(t *testing.T) {
|
||||
cause := errors.New("broken embedded schema")
|
||||
client := &scriptedValidationClient{replies: []validationReply{
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}},
|
||||
{err: &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "model setup failed", Cause: cause}},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}},
|
||||
}}
|
||||
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
|
||||
validator.SetTokenEstimator(perItemEstimator{})
|
||||
|
||||
@@ -2,6 +2,7 @@ package validators
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
@@ -9,6 +10,7 @@ import (
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/jsonnumber"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/stagename"
|
||||
@@ -58,8 +60,8 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
if req.LLMClient == nil {
|
||||
return Result{}, fmt.Errorf("LLM-backed validator %q requires a structured LLM client", v.name)
|
||||
}
|
||||
validationReq, immediate := BuildLLMValidationRequest(v.name, v.validatorType, req)
|
||||
if len(validationReq.Items) == 0 {
|
||||
items, immediate := buildValidationItems(req)
|
||||
if len(items) == 0 {
|
||||
all := append([]Decision(nil), immediate...)
|
||||
sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex })
|
||||
if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil {
|
||||
@@ -73,7 +75,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
maxTokens = req.Settings.Validation.MaxPromptTokens
|
||||
}
|
||||
|
||||
batches, oversized, err := prepareValidationBatches(validationReq.Items, maxTokens, v.estimator)
|
||||
batches, oversized, err := prepareValidationBatches(items, maxTokens, v.estimator)
|
||||
operations := make([]outcomes.Operation, 0)
|
||||
llmDecisions := make([]Decision, 0)
|
||||
partialResult := func() Result {
|
||||
@@ -239,7 +241,7 @@ func errPayload(err error) any {
|
||||
return map[string]any{"error": err.Error()}
|
||||
}
|
||||
|
||||
func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorType, req Request) (LLMValidationRequest, []Decision) {
|
||||
func buildValidationItems(req Request) ([]LLMValidationItem, []Decision) {
|
||||
items := make([]LLMValidationItem, 0, len(req.CandidateProposal))
|
||||
immediate := make([]Decision, 0)
|
||||
req.Section = req.sectionLookup()
|
||||
@@ -268,7 +270,6 @@ func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorT
|
||||
neighbors, glossaryEntries := glossary.validationContext(segments, position, preview.CorrectedSegmentText)
|
||||
items = append(items, LLMValidationItem{
|
||||
CorrectionIndex: p.ProposalIndex,
|
||||
SegmentID: p.TargetSegmentID,
|
||||
Speaker: seg.Speaker,
|
||||
OriginalText: p.OriginalText,
|
||||
CorrectedText: p.CorrectedText,
|
||||
@@ -280,31 +281,25 @@ func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorT
|
||||
})
|
||||
}
|
||||
|
||||
return LLMValidationRequest{
|
||||
ValidatorName: validatorName,
|
||||
ValidatorType: validatorType,
|
||||
ModuleKey: req.ModuleKey,
|
||||
ModuleInstance: req.ModuleInstance,
|
||||
ReplacementPolicy: string(req.ReplacementPolicy),
|
||||
Items: items,
|
||||
}, immediate
|
||||
return items, immediate
|
||||
}
|
||||
|
||||
type validationReconciliation struct {
|
||||
Decisions []Decision `json:"decisions"`
|
||||
UnavailableIndices []int `json:"unavailable_indices,omitempty"`
|
||||
IgnoredIndices []int `json:"ignored_indices,omitempty"`
|
||||
Decisions []Decision `json:"decisions"`
|
||||
UnavailableIndices []int `json:"unavailable_indices,omitempty"`
|
||||
IgnoredIndices []json.Number `json:"ignored_indices,omitempty"`
|
||||
}
|
||||
|
||||
func reconcileLLMResponse(items []LLMValidationItem, response LLMValidationResponse) validationReconciliation {
|
||||
result := validationReconciliation{Decisions: make([]Decision, 0, len(items))}
|
||||
seen := make(map[int][]LLMValidationDecision, len(items))
|
||||
for _, d := range response.Validations {
|
||||
if d.CorrectionIndex < 1 || d.CorrectionIndex > len(items) {
|
||||
index, ok := jsonnumber.PositiveIndex(d.CorrectionIndex.String(), len(items))
|
||||
if !ok {
|
||||
result.IgnoredIndices = append(result.IgnoredIndices, d.CorrectionIndex)
|
||||
continue
|
||||
}
|
||||
seen[d.CorrectionIndex] = append(seen[d.CorrectionIndex], d)
|
||||
seen[index] = append(seen[index], d)
|
||||
}
|
||||
|
||||
for i, item := range items {
|
||||
|
||||
@@ -4,6 +4,8 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -74,7 +76,7 @@ func mk(index int, orig, corr string) proposals.EnrichedCorrectionProposal {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildLLMValidationRequestPreservesLiteralDeletionAndSpacing(t *testing.T) {
|
||||
func TestValidationPayloadPreservesLiteralDeletionAndSpacing(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, segment, original, replacement, result string
|
||||
}{
|
||||
@@ -84,27 +86,25 @@ func TestBuildLLMValidationRequestPreservesLiteralDeletionAndSpacing(t *testing.
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(7, tc.original, tc.replacement)})
|
||||
req.WorkingTranscript.Segments[0].Text = tc.segment
|
||||
payload, immediate := BuildLLMValidationRequest("editorial_review", LLMValidatorTypeEditorialReview, req)
|
||||
if len(immediate) != 0 || len(payload.Items) != 1 {
|
||||
payload, immediate := buildValidationItems(req)
|
||||
if len(immediate) != 0 || len(payload) != 1 {
|
||||
t.Fatalf("expected previewable request: %+v, %+v", payload, immediate)
|
||||
}
|
||||
raw, err := json.Marshal(payload)
|
||||
variables, err := BuildValidationVariables(payload, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var decoded struct {
|
||||
ValidatorName string `json:"validator_name"`
|
||||
Corrections []map[string]any `json:"corrections"`
|
||||
}
|
||||
raw := []byte(variables["PayloadJSON"])
|
||||
var decoded []map[string]any
|
||||
if err := json.Unmarshal(raw, &decoded); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if decoded.ValidatorName != "editorial_review" || len(decoded.Corrections) != 1 {
|
||||
t.Fatalf("wrong validator identity or item count: %s", raw)
|
||||
if len(decoded) != 1 {
|
||||
t.Fatalf("wrong item count: %s", raw)
|
||||
}
|
||||
item := decoded.Corrections[0]
|
||||
item := decoded[0]
|
||||
value, present := item["corrected_text"]
|
||||
if !present || value != tc.replacement || item["original_text"] != tc.original || item["original_segment_text"] != tc.segment || item["corrected_segment_text"] != tc.result || item["correction_index"] != float64(7) {
|
||||
if !present || value != tc.replacement || item["original_text"] != tc.original || item["original_segment_text"] != tc.segment || item["corrected_segment_text"] != tc.result || item["correction_index"] != float64(1) {
|
||||
t.Fatalf("literal spans or full segment text lost: %s", raw)
|
||||
}
|
||||
})
|
||||
@@ -148,8 +148,8 @@ func TestChunkLLMValidationItemsMultipleBatchesStableNoDropNoDup(t *testing.T) {
|
||||
|
||||
func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) {
|
||||
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
|
||||
{CorrectionIndex: 1, Approved: true, Reason: ""},
|
||||
{CorrectionIndex: 2, Approved: false, Reason: "bad"},
|
||||
{CorrectionIndex: "1", Approved: true, Reason: ""},
|
||||
{CorrectionIndex: "2", Approved: false, Reason: "bad"},
|
||||
}}}}
|
||||
v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
|
||||
if err != nil {
|
||||
@@ -203,10 +203,10 @@ func TestLLMBackedValidatorProviderMalformedEnvelopePropagates(t *testing.T) {
|
||||
|
||||
func TestLLMBackedValidatorReconcilesDecisionIdentities(t *testing.T) {
|
||||
approve := func(index int) LLMValidationDecision {
|
||||
return LLMValidationDecision{CorrectionIndex: index, Approved: true, Reason: "accepted"}
|
||||
return LLMValidationDecision{CorrectionIndex: json.Number(strconv.Itoa(index)), Approved: true, Reason: "accepted"}
|
||||
}
|
||||
reject := func(index int) LLMValidationDecision {
|
||||
return LLMValidationDecision{CorrectionIndex: index, Reason: "editorial concern"}
|
||||
return LLMValidationDecision{CorrectionIndex: json.Number(strconv.Itoa(index)), Reason: "editorial concern"}
|
||||
}
|
||||
type expectedDecision struct {
|
||||
index int
|
||||
@@ -218,13 +218,13 @@ func TestLLMBackedValidatorReconcilesDecisionIdentities(t *testing.T) {
|
||||
entries []LLMValidationDecision
|
||||
want []expectedDecision
|
||||
unavailable []int
|
||||
ignored []int
|
||||
ignored []json.Number
|
||||
}{
|
||||
{"internal index is not a local reference", []LLMValidationDecision{approve(7), approve(2), reject(3)}, []expectedDecision{{7, false, ReasonMissingValidationDecision}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, []int{7}, []int{7}},
|
||||
{"missing and extra", []LLMValidationDecision{approve(1), reject(3), approve(99)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonMissingValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, []int{99}},
|
||||
{"internal index is not a local reference", []LLMValidationDecision{approve(7), approve(2), reject(3)}, []expectedDecision{{7, false, ReasonMissingValidationDecision}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, []int{7}, []json.Number{"7"}},
|
||||
{"missing and extra", []LLMValidationDecision{approve(1), reject(3), approve(99)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonMissingValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, []json.Number{"99"}},
|
||||
{"conflicting duplicate", []LLMValidationDecision{approve(1), approve(2), reject(2), reject(3)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonDuplicateValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
|
||||
{"identical duplicate", []LLMValidationDecision{approve(1), approve(2), approve(2), reject(3)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonDuplicateValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
|
||||
{"extra only", []LLMValidationDecision{approve(99), reject(3), approve(1), approve(99), approve(2)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, []int{99, 99}},
|
||||
{"extra only", []LLMValidationDecision{approve(99), reject(3), approve(1), approve(99), approve(2)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, []json.Number{"99", "99"}},
|
||||
{"empty", nil, []expectedDecision{{7, false, ReasonMissingValidationDecision}, {11, false, ReasonMissingValidationDecision}, {18, false, ReasonMissingValidationDecision}}, []int{7, 11, 18}, nil},
|
||||
{"reordered", []LLMValidationDecision{reject(3), approve(2), approve(1)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, nil},
|
||||
} {
|
||||
@@ -264,7 +264,7 @@ func TestLLMBackedValidatorReconcilesDecisionIdentities(t *testing.T) {
|
||||
t.Fatalf("supplied model decisions lost from diagnostic: %+v", decoded)
|
||||
}
|
||||
reconciliation := payload["reconciliation"].(validationReconciliation)
|
||||
if !equalIndices(reconciliation.UnavailableIndices, tc.unavailable) || !equalIndices(reconciliation.IgnoredIndices, tc.ignored) || len(reconciliation.Decisions) != len(tc.want) {
|
||||
if !equalIndices(reconciliation.UnavailableIndices, tc.unavailable) || !slices.Equal(reconciliation.IgnoredIndices, tc.ignored) || len(reconciliation.Decisions) != len(tc.want) {
|
||||
t.Fatalf("diagnostic reconciliation=%+v", reconciliation)
|
||||
}
|
||||
})
|
||||
@@ -285,7 +285,7 @@ func equalIndices(a, b []int) bool {
|
||||
|
||||
func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *testing.T) {
|
||||
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
|
||||
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
|
||||
{CorrectionIndex: "1", Approved: true, Reason: "ok"},
|
||||
}}}}
|
||||
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
|
||||
huge := strings.Repeat("gestures ", 200)
|
||||
@@ -331,7 +331,7 @@ func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *tes
|
||||
|
||||
func TestLLMBackedValidatorDiagnosticsIncludeSchemaMetadata(t *testing.T) {
|
||||
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
|
||||
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
|
||||
{CorrectionIndex: "1", Approved: true, Reason: "ok"},
|
||||
}}}}
|
||||
writer := &captureValidationDiagnosticsWriter{}
|
||||
v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
|
||||
@@ -370,7 +370,7 @@ func TestEveryValidationTypeSelectsEmbeddedPromptAndVariables(t *testing.T) {
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(string(tc.kind), func(t *testing.T) {
|
||||
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}}}
|
||||
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}}}
|
||||
validator, err := NewLLMBackedValidator(string(tc.kind), tc.kind)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -395,7 +395,7 @@ func TestEveryValidationTypeSelectsEmbeddedPromptAndVariables(t *testing.T) {
|
||||
func TestValidationDiagnosticFailureDoesNotChangeDecision(t *testing.T) {
|
||||
cause := errors.New("diagnostic storage unavailable")
|
||||
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
||||
req.LLMClient = &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}}}
|
||||
req.LLMClient = &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}}}
|
||||
sink := &captureValidationDiagnosticsWriter{err: cause}
|
||||
req.Interactions = sink
|
||||
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
|
||||
|
||||
@@ -62,6 +62,25 @@ type Decision struct {
|
||||
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
|
||||
}
|
||||
|
||||
// EffectiveDisposition resolves an omitted disposition from the declared
|
||||
// approval and detects inconsistent explicit decisions.
|
||||
func (d Decision) EffectiveDisposition() (DecisionDisposition, error) {
|
||||
switch d.Disposition {
|
||||
case "":
|
||||
if d.Approved {
|
||||
return DispositionApproved, nil
|
||||
}
|
||||
return DispositionRejected, nil
|
||||
case DispositionApproved, DispositionRejected, DispositionUnavailable:
|
||||
if d.Approved == (d.Disposition == DispositionApproved) {
|
||||
return d.Disposition, nil
|
||||
}
|
||||
return "", fmt.Errorf("contradictory approval and disposition %q for proposal %d", d.Disposition, d.ProposalIndex)
|
||||
default:
|
||||
return "", fmt.Errorf("invalid decision disposition %q for proposal %d", d.Disposition, d.ProposalIndex)
|
||||
}
|
||||
}
|
||||
|
||||
// DecisionDisposition separates normal editorial rejection from a candidate
|
||||
// excluded because its model operation could not provide a trustworthy decision.
|
||||
type DecisionDisposition string
|
||||
@@ -80,7 +99,10 @@ type Result struct {
|
||||
Operations []outcomes.Operation `json:"operations,omitempty"`
|
||||
}
|
||||
|
||||
// Validator is the deterministic runtime validator interface.
|
||||
// Validator evaluates one section's candidates. An instance may be called
|
||||
// concurrently across sections and may keep synchronized state. Request data
|
||||
// is borrowed for the call; inputs must be copied before modification or
|
||||
// retention. Request.Preview may populate the caller's pass-local cache.
|
||||
type Validator interface {
|
||||
Name() string
|
||||
Validate(ctx context.Context, req Request) (Result, error)
|
||||
|
||||
@@ -54,3 +54,21 @@ func (r Request) Preview(candidate proposals.EnrichedCorrectionProposal) proposa
|
||||
}
|
||||
return preview
|
||||
}
|
||||
|
||||
// AcceptsIndependentCandidate decides which evidence can be submitted when
|
||||
// repair needs findings from each applicable validator.
|
||||
func (r Request) AcceptsIndependentCandidate(validatorName string, modelDriven bool, candidate proposals.EnrichedCorrectionProposal) bool {
|
||||
hasSpan := candidate.OriginalText != ""
|
||||
switch validatorName {
|
||||
case "proposal_shape":
|
||||
return true
|
||||
case "no_effect":
|
||||
return hasSpan
|
||||
case "original_text_presence":
|
||||
return hasSpan && candidate.TargetSegmentID > 0
|
||||
case "protected_terms", "non_empty_resulting_segment":
|
||||
return r.Preview(candidate).HasResult()
|
||||
default:
|
||||
return !modelDriven || r.Preview(candidate).Applicable
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user