Validate internal decisions and reconcile exact model references

This commit is contained in:
2026-11-16 14:11:02 +00:00
parent 475557e4de
commit 0c144fd5a5
46 changed files with 544 additions and 376 deletions

View File

@@ -53,9 +53,17 @@ assessment is unavailable; an unavailable assessment alone does not. Revised
candidates run the applicable checks again. A zero repair budget retains the
original shortlist flow.
Decision cardinality is enforced inside Audita: each candidate proposal must
receive exactly one outcome per validator it reaches. The runner still rejects
duplicate, missing, or unknown identities in an internal validator result.
The shared validator port lives in `internal/framework/validators`. Instances
may run concurrently across sections and can keep synchronized state. Request
data is borrowed for the duration of the call. Copy data before modifying or
retaining it.
The runner checks decision identity and disposition before allowing a candidate
to advance. A normal return supplies one decision per submitted candidate. A
fatal return may provide a checked subset as evidence. Contradictory approval
and disposition are a fatal contract failure. The recorded disposition governs
approval, and each record carries the effective execution class, including
instance overrides, for initial validation and revalidation.
## LLM Validator Batching
@@ -89,8 +97,11 @@ Reconciliation makes no semantic retry or extra model call; guided recovery rema
[deferred](../roadmap/future.md#domain-aware-response-recovery).
Debug request metadata records `proposal_indices` in local-reference order
(reference 1 is the first entry). Raw decoded responses and ignored indices use
local references; reconciled decisions and unavailable indices use internal
(reference 1 is the first entry). Decoding preserves the supplied numeric
tokens, including integral decimal or exponent spellings. Reconciliation
checks the batch range before converting to a local integer, so an extra
reference beyond the native integer range can be ignored. Raw decoded responses
and ignored indices use local references. Reconciled decisions use internal
proposal indices. Batch token estimates use the same locally numbered payload
as prompt encoding.

View File

@@ -355,6 +355,13 @@ beside an out-of-range reference. Cover equivalent numeric spellings,
contradictory internal results, and class override from execution through the
record supplied to Stage 17. Update [validator internals](../internal/validators.md).
**Complete.** Validators share one port and applicability owner. The runner
rejects contradictory decisions and records the effective class on initial and
repair validation evidence. Controlled provider responses verified exact
numeric decoding, equivalent spellings, duplicate and missing decisions, and
a usable decision alongside a reference above the native integer range. The
CLI build and full offline suite passed.
## Stage 10 — Reduce validation context work
Audit #36.

View File

@@ -0,0 +1,82 @@
// Package jsonnumber converts exact integral JSON tokens without float rounding.
package jsonnumber
import (
"strconv"
"strings"
)
// Int accepts lexical integers and equivalent decimal and exponent spellings
// in the native integer range. value must be a valid JSON number token.
func Int(value string) (int, bool) {
if integer, err := strconv.Atoi(value); err == nil {
return integer, true
}
digits, ok := integralDecimal(value)
if !ok {
return 0, false
}
integer, err := strconv.Atoi(digits)
return integer, err == nil
}
// PositiveIndex accepts only references in [1, maximum]. Comparison with the
// requested range precedes conversion to the local integer index.
func PositiveIndex(value string, maximum int) (int, bool) {
if strings.ContainsAny(value, ".eE") {
var ok bool
if value, ok = integralDecimal(value); !ok {
return 0, false
}
}
if maximum < 1 || value == "" || value[0] < '1' || value[0] > '9' {
return 0, false
}
limit := strconv.Itoa(maximum)
if len(value) > len(limit) || (len(value) == len(limit) && value > limit) {
return 0, false
}
index, err := strconv.Atoi(value)
return index, err == nil
}
func integralDecimal(value string) (string, bool) {
if value == "" || (value[0] != '-' && (value[0] < '0' || value[0] > '9')) || !strings.ContainsAny(value, ".eE") {
return "", false
}
digits, negative := strings.CutPrefix(value, "-")
var exponentText string
if i := strings.IndexAny(digits, "eE"); i >= 0 {
exponentText, digits = digits[i+1:], digits[:i]
}
whole, fraction, _ := strings.Cut(digits, ".")
digits = strings.TrimLeft(whole+fraction, "0")
if digits == "" {
return "0", true
}
var exponent int
if exponentText != "" {
var err error
exponent, err = strconv.Atoi(exponentText)
if err != nil || exponent < -len(digits) || exponent > len(fraction)+19 {
return "", false
}
}
shift := exponent - len(fraction)
if shift < 0 {
// A negative shift is integral only if every discarded digit is zero.
if -shift > len(digits)-len(strings.TrimRight(digits, "0")) {
return "", false
}
digits = digits[:len(digits)+shift]
} else {
if len(digits)+shift > 19 {
return "", false
}
digits += strings.Repeat("0", shift)
}
if negative {
digits = "-" + digits
}
return digits, true
}

View File

@@ -6,8 +6,8 @@ import (
"fmt"
"math"
"slices"
"strconv"
"strings"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/jsonnumber"
)
type SourceSegment struct {
@@ -114,60 +114,13 @@ func extractSegments(raw []byte) (json.RawMessage, error) {
type sourceID int
func (id *sourceID) UnmarshalJSON(raw []byte) error {
spelling := string(raw)
if value, err := strconv.Atoi(spelling); err == nil {
*id = sourceID(value)
return nil
}
if value, ok := integralDecimal(spelling); ok {
if value, ok := jsonnumber.Int(string(raw)); ok {
*id = sourceID(value)
return nil
}
return json.Unmarshal(raw, (*int)(id))
}
func integralDecimal(value string) (int, bool) {
if value == "" || (value[0] != '-' && (value[0] < '0' || value[0] > '9')) || !strings.ContainsAny(value, ".eE") {
return 0, false
}
digits, negative := strings.CutPrefix(value, "-")
var exponentText string
if i := strings.IndexAny(digits, "eE"); i >= 0 {
exponentText, digits = digits[i+1:], digits[:i]
}
whole, fraction, _ := strings.Cut(digits, ".")
digits = strings.TrimLeft(whole+fraction, "0")
if digits == "" {
return 0, true
}
var exponent int
if exponentText != "" {
var err error
exponent, err = strconv.Atoi(exponentText)
if err != nil || exponent < -len(digits) || exponent > len(fraction)+19 {
return 0, false
}
}
shift := exponent - len(fraction)
if shift < 0 {
// A negative shift is integral only if every discarded digit is zero.
if -shift > len(digits)-len(strings.TrimRight(digits, "0")) {
return 0, false
}
digits = digits[:len(digits)+shift]
} else {
if len(digits)+shift > 19 {
return 0, false
}
digits += strings.Repeat("0", shift)
}
if negative {
digits = "-" + digits
}
integer, err := strconv.Atoi(digits)
return integer, err == nil
}
func validateSourceSegments(segments []SourceSegment) error {
for i, s := range segments {
segLabel := segmentLabel(i, s.ID)

View File

@@ -9,6 +9,7 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
)
// testChunkProposalHarness is a test-only helper that composes existing
@@ -77,7 +78,7 @@ func (m deterministicFakeModule) ReplacementPolicy() proposals.ReplacementPolicy
return m.replacementPolicy
}
func (m deterministicFakeModule) Validators() []Validator { return nil }
func (m deterministicFakeModule) Validators() []validators.Validator { return nil }
func (m deterministicFakeModule) Propose(ctx context.Context, req ProposalRequest) (ProposalResult, error) {
_ = ctx

View File

@@ -22,16 +22,10 @@ type StructuredLLMClient = structuredoutput.Client
type TranscriptModule interface {
Key() string
ReplacementPolicy() proposals.ReplacementPolicy
Validators() []Validator
Validators() []validators.Validator
Propose(ctx context.Context, req ProposalRequest) (ProposalResult, error)
}
// Validator evaluates candidate proposals and returns one decision per proposal index.
type Validator interface {
Name() string
Validate(ctx context.Context, req ValidationRequest) (validators.Result, error)
}
type StructuredCompletionRequest = structuredoutput.Request
type StructuredCompletionResponse = structuredoutput.Response
@@ -98,9 +92,6 @@ type ProposalResult struct {
Operations []outcomes.Operation `json:"operations,omitempty"`
}
// ValidationRequest is the input to validator execution.
type ValidationRequest = validators.Request
// ResolveModuleRunSpecs derives isolated consumer views from resolved settings.
func ResolveModuleRunSpecs(cfg *config.Config) []ModuleRunSpec {
if cfg == nil {

View File

@@ -30,7 +30,7 @@ type fakeValidator struct{}
func (f *fakeValidator) Name() string { return "fake-validator" }
func (f *fakeValidator) Validate(ctx context.Context, req ValidationRequest) (validators.Result, error) {
func (f *fakeValidator) Validate(ctx context.Context, req validators.Request) (validators.Result, error) {
_ = ctx
decisions := make([]validators.Decision, len(req.CandidateProposal))
for i, proposal := range req.CandidateProposal {
@@ -47,8 +47,8 @@ func (f *fakeModule) ReplacementPolicy() proposals.ReplacementPolicy {
return proposals.ReplacementPolicyRequireUnique
}
func (f *fakeModule) Validators() []Validator {
return []Validator{&fakeValidator{}}
func (f *fakeModule) Validators() []validators.Validator {
return []validators.Validator{&fakeValidator{}}
}
func (f *fakeModule) Propose(ctx context.Context, req ProposalRequest) (ProposalResult, error) {
@@ -63,7 +63,7 @@ func (f *fakeModule) Propose(ctx context.Context, req ProposalRequest) (Proposal
func TestInterfaceContractsCompileWithFakes(t *testing.T) {
var _ StructuredLLMClient = (*fakeLLMClient)(nil)
var _ Validator = (*fakeValidator)(nil)
var _ validators.Validator = (*fakeValidator)(nil)
var _ TranscriptModule = (*fakeModule)(nil)
module := &fakeModule{}

View File

@@ -3,6 +3,7 @@ package modules
import (
"context"
"errors"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
"strings"
"testing"
@@ -20,7 +21,7 @@ func (m noopModule) Key() string { return m.key }
func (m noopModule) ReplacementPolicy() proposals.ReplacementPolicy {
return proposals.ReplacementPolicyRequireUnique
}
func (m noopModule) Validators() []contracts.Validator { return nil }
func (m noopModule) Validators() []frameworkvalidators.Validator { return nil }
func (m noopModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
_ = ctx
_ = req

View File

@@ -2,7 +2,9 @@ package processreport
import (
"context"
"encoding/json"
"errors"
"strconv"
"strings"
"testing"
@@ -16,13 +18,15 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
)
type partialModule struct{ validator contracts.Validator }
type partialModule struct{ validator validators.Validator }
func (m partialModule) Key() string { return "grammar" }
func (m partialModule) ReplacementPolicy() proposals.ReplacementPolicy {
return proposals.ReplacementPolicyRequireUnique
}
func (m partialModule) Validators() []contracts.Validator { return []contracts.Validator{m.validator} }
func (m partialModule) Validators() []validators.Validator {
return []validators.Validator{m.validator}
}
func (m partialModule) ModuleForSpec(contracts.ModuleRunSpec) (contracts.TranscriptModule, error) {
return m, nil
}
@@ -55,7 +59,7 @@ func (c *interruptedValidationClient) CompleteStructured(_ context.Context, req
*target.(*validators.LLMValidationResponse) = validators.LLMValidationResponse{}
return structuredoutput.Response{}, nil
}
*target.(*validators.LLMValidationResponse) = validators.LLMValidationResponse{Validations: []validators.LLMValidationDecision{{CorrectionIndex: index, Approved: index == 1, Reason: "completed assessment"}}}
*target.(*validators.LLMValidationResponse) = validators.LLMValidationResponse{Validations: []validators.LLMValidationDecision{{CorrectionIndex: json.Number(strconv.Itoa(index)), Approved: index == 1, Reason: "completed assessment"}}}
return structuredoutput.Response{}, nil
}

View File

@@ -104,7 +104,7 @@ func TestValidatorFailureCancelsActiveAndPendingProviderWork(t *testing.T) {
}, &out)
return validators.Result{}, err
}}
module := fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{validator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
module := fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{validator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "hello", CorrectedText: "Hello"}}, nil
}}
sections := []chunking.Section{}

View File

@@ -1,48 +1,19 @@
package runner
import (
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
)
// feedbackCandidates submits only candidates for which the validator has usable evidence.
func feedbackCandidates(input validateSectionCandidatesInput, validator contracts.Validator, noEffect map[int]bool, evidence contracts.ValidationRequest) []proposals.EnrichedCorrectionProposal {
func feedbackCandidates(input validateSectionCandidatesInput, validatorName string, modelDriven bool, noEffect map[int]bool, evidence validators.Request) []proposals.EnrichedCorrectionProposal {
var result []proposals.EnrichedCorrectionProposal
for _, candidate := range input.SectionEnriched {
if noEffect[candidate.ProposalIndex] {
continue
}
usableSpan := candidate.OriginalText != ""
target := candidate.TargetSegmentID > 0
var applicable bool
switch validator.Name() {
case "proposal_shape":
applicable = true
case "no_effect":
applicable = usableSpan
case "original_text_presence":
applicable = usableSpan && target
case "protected_terms":
applicable = usableSpan
if input.Spec.ModuleKey == modulecatalog.KeyGlossary || input.Spec.ModuleKey == modulecatalog.KeySpokenWord {
preview := evidence.Preview(candidate)
applicable = preview.Applicable || preview.SkipReason == proposals.SkipReasonEmptyResultingText
}
case "non_empty_resulting_segment":
preview := evidence.Preview(candidate)
applicable = preview.Applicable || preview.SkipReason == proposals.SkipReasonEmptyResultingText
default:
if validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked {
applicable = evidence.Preview(candidate).Applicable
} else {
applicable = true
}
}
if applicable {
if evidence.AcceptsIndependentCandidate(validatorName, modelDriven, candidate) {
result = append(result, candidate)
}
}

View File

@@ -25,7 +25,7 @@ func TestFeedbackCollectsIndependentFindingsAndUsableSemanticInput(t *testing.T)
return proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: target, OriginalText: original, CorrectedText: replacement}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index}}
}
var seen []int
semantic := validatormetadata.Wrap(fakeValidator{name: "semantic", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
semantic := validatormetadata.Wrap(fakeValidator{name: "semantic", validateF: func(req validators.Request) (validators.Result, error) {
var decisions []validators.Decision
for _, p := range req.CandidateProposal {
seen = append(seen, p.ProposalIndex)
@@ -33,7 +33,7 @@ func TestFeedbackCollectsIndependentFindingsAndUsableSemanticInput(t *testing.T)
}
return validators.Result{Decisions: decisions}, nil
}}, validatormetadata.ExecutionClassLLMBacked)
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh teh cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, semantic}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate(0, 1, "cat", "dog"), candidate(1, 0, "teh", "the"), candidate(2, 1, "teh", "the"), candidate(3, 1, "cat", "cat")}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh teh cat"}}}, Validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, semantic}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate(0, 1, "cat", "dog"), candidate(1, 0, "teh", "the"), candidate(2, 1, "teh", "the"), candidate(3, 1, "cat", "cat")}}
result, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
@@ -58,14 +58,14 @@ func TestFeedbackKeepsUnavailableAndZeroBudgetShortlists(t *testing.T) {
for _, budget := range []int{0, 1} {
t.Run(fmt.Sprint(budget), func(t *testing.T) {
var later int
first := fakeValidator{name: "first", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
first := fakeValidator{name: "first", validateF: func(req validators.Request) (validators.Result, error) {
var decisions []validators.Decision
for _, p := range req.CandidateProposal {
decisions = append(decisions, validators.Decision{ProposalIndex: p.ProposalIndex, Disposition: validators.DispositionUnavailable, ReasonCode: "unavailable"})
}
return validators.Result{Decisions: decisions}, nil
}}
second := fakeValidator{name: "second", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
second := fakeValidator{name: "second", validateF: func(req validators.Request) (validators.Result, error) {
later = len(req.CandidateProposal)
var decisions []validators.Decision
for _, p := range req.CandidateProposal {
@@ -73,7 +73,7 @@ func TestFeedbackKeepsUnavailableAndZeroBudgetShortlists(t *testing.T) {
}
return validators.Result{Decisions: decisions}, nil
}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{first, second}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{first, second}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}}
result, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
@@ -95,8 +95,8 @@ func TestRepairFindingsUseSafeUnavailableEvidence(t *testing.T) {
func TestRevalidationArtifactsUseProposalAndAttemptCoordinates(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}},
}}
validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
if err != nil {
@@ -108,7 +108,7 @@ func TestRevalidationArtifactsUseProposalAndAttemptCoordinates(t *testing.T) {
var paths []string
for _, index := range []int{2, 3} {
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar_1", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validator}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, ValidationLLMClient: client, Interactions: sink, SectionIndex: &section, RepairProposalIndex: &index, RepairAttempt: &attempt}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar_1", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{validator}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, ValidationLLMClient: client, Interactions: sink, SectionIndex: &section, RepairProposalIndex: &index, RepairAttempt: &attempt}
result, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
@@ -149,11 +149,11 @@ func TestRevalidationArtifactsUseProposalAndAttemptCoordinates(t *testing.T) {
func TestFeedbackReportsProtectionAndBlankResult(t *testing.T) {
identity := glossary.Identity{Collection: "party", ID: "jesters"}
protected := fakeValidator{name: "protected_terms", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
protected := fakeValidator{name: "protected_terms", validateF: func(req validators.Request) (validators.Result, error) {
return validators.ValidateProtectedTerms(req)
}}
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "Jesters", CorrectedText: ""}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "spoken_word", ProtectedTerms: []glossary.Term{{Spelling: "Jesters", Identities: []glossary.Identity{identity}}}, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Jesters"}}}, Validators: []contracts.Validator{protected, validators.NonEmptyResultingSegmentValidator{}}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "spoken_word", ProtectedTerms: []glossary.Term{{Spelling: "Jesters", Identities: []glossary.Identity{identity}}}, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Jesters"}}}, Validators: []validators.Validator{protected, validators.NonEmptyResultingSegmentValidator{}}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}}
result, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)

View File

@@ -2,6 +2,7 @@ package runner
import (
"context"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
@@ -27,7 +28,7 @@ func (m orderedModule) Key() string { return m.spec.ModuleKey }
func (m orderedModule) ReplacementPolicy() proposals.ReplacementPolicy {
return proposals.ReplacementPolicyRequireUnique
}
func (m orderedModule) Validators() []contracts.Validator { return nil }
func (m orderedModule) Validators() []frameworkvalidators.Validator { return nil }
func (m orderedModule) Propose(_ context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
m.seen <- req
original, corrected := "party", "Party"

View File

@@ -24,6 +24,7 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
)
type repairClient struct {
@@ -52,7 +53,7 @@ func TestRepairBranchesRetargetAndPreserveOtherApprovals(t *testing.T) {
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 4}},
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 0, OriginalText: "x", CorrectedText: "y"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 5}},
}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}, {ID: 2, Text: "uh hello"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.NonEmptyResultingSegmentValidator{}}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}, {ID: 2, Text: "uh hello"}}}, Validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.NonEmptyResultingSegmentValidator{}}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{}}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
@@ -92,7 +93,7 @@ func TestRepairRequestCarriesLiteralPreview(t *testing.T) {
Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}},
Policy: tc.policy,
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 81, Text: tc.segment}}},
Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.NonEmptyResultingSegmentValidator{}, rejectUntilRepair()},
Validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.NonEmptyResultingSegmentValidator{}, rejectUntilRepair()},
SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate},
Exchange: &structuredoutput.Exchange{},
}
@@ -131,7 +132,7 @@ func TestRepairRequestRefreshesPreviewAfterRetarget(t *testing.T) {
`{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`,
}}
var validatedIDs []int
reject := fakeValidator{name: "editorial_review", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
reject := fakeValidator{name: "editorial_review", validateF: func(req validators.Request) (validators.Result, error) {
validatedIDs = append(validatedIDs, req.CandidateProposal[0].TargetSegmentID)
return validators.Result{Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Disposition: validators.DispositionRejected, ReasonCode: "llm_rejected", Message: "The context does not support this edit."}}}, nil
}}
@@ -139,9 +140,10 @@ func TestRepairRequestRefreshesPreviewAfterRetarget(t *testing.T) {
Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 2}}},
Policy: proposals.ReplacementPolicyRequireUnique,
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 81, Text: "cat"}, {ID: 104, Text: "uh hello"}}},
Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, reject},
Validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validatormetadata.Wrap(reject, validatormetadata.ExecutionClassDeterministic)},
SectionEnriched: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 81, OriginalText: "cat", CorrectedText: "dog"}}},
Exchange: &structuredoutput.Exchange{},
Interactions: llm.NewDiagnosticsWriter(t.TempDir()),
}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
@@ -154,6 +156,15 @@ func TestRepairRequestRefreshesPreviewAfterRetarget(t *testing.T) {
if len(validatedIDs) != 2 || validatedIDs[0] != 81 || validatedIDs[1] != 104 {
t.Fatalf("validated segment IDs: %v", validatedIDs)
}
for _, decisions := range [][]ValidatorDecisionRecord{initial.decisions, branches[0].Attempts[0].Decisions} {
if len(decisions) == 0 {
t.Fatal("missing decision evidence")
}
record := decisions[len(decisions)-1]
if record.ValidatorName != "editorial_review" || record.ExecutionClass != validatormetadata.ExecutionClassDeterministic {
t.Fatalf("lost execution class override: %+v", record)
}
}
for i, want := range []struct{ before, after string }{{"cat", "dog"}, {"uh hello", "hello"}} {
var payload proposal_generation.RepairRequest
if err := json.Unmarshal([]byte(client.calls[i].Variables["RepairRequestJSON"]), &payload); err != nil {
@@ -183,7 +194,7 @@ func TestRepairBranchTerminalOutcomes(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
client := &repairClient{t: t, outputs: tc.outputs, err: tc.err}
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: tc.budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: tc.budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
@@ -211,7 +222,7 @@ func TestRunnerAppliesRevisionsInOriginalOrder(t *testing.T) {
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":2,"decision":{"action":"revise","correction":{"id":1,"original_text":"cat","corrected_text":"fox"}}}`,
`{"proposal_ref":3,"decision":{"action":"revise","correction":{"id":1,"original_text":"bird","corrected_text":"owl"}}}`,
}}
module := repairModule{outcomeModule{fakeModule: fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, rejectUntilRepair(0)}}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
module := repairModule{outcomeModule{fakeModule: fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, rejectUntilRepair(0)}}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
return contracts.ProposalResult{Proposals: candidates, SegmentReferences: []int{1, 1, 1}, Exchange: &structuredoutput.Exchange{}}, nil
}}}
runner := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}})
@@ -244,7 +255,7 @@ func TestRepairRevalidatesEarlierApprovalAndRetainsUnrelatedApprovalOnFailure(t
}{{"revision", false}, {"provider failure", true}} {
t.Run(tc.name, func(t *testing.T) {
calls := 0
check := fakeValidator{name: "custom", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
check := fakeValidator{name: "custom", validateF: func(req validators.Request) (validators.Result, error) {
var decisions []validators.Decision
for _, candidate := range req.CandidateProposal {
calls++
@@ -256,7 +267,7 @@ func TestRepairRevalidatesEarlierApprovalAndRetainsUnrelatedApprovalOnFailure(t
if tc.failure {
client.err = &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonProviderFailed, Message: "failed"}
}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat bird"}}}, Validators: []contracts.Validator{check, rejectUntilRepair(0)}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat bird"}}}, Validators: []validators.Validator{check, rejectUntilRepair(0)}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{}}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
@@ -279,7 +290,7 @@ func TestRepairRevalidatesEarlierApprovalAndRetainsUnrelatedApprovalOnFailure(t
func TestRepairInterruptionRetainsAttempt(t *testing.T) {
client := &repairClient{t: t, err: &structuredoutput.Failure{Class: structuredoutput.FailureCanceled, Reason: outcomes.ReasonCanceled, Message: "canceled", Cause: context.Canceled}}
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
@@ -297,7 +308,7 @@ func TestRepairArtifactsAreUnique(t *testing.T) {
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "fox"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}},
}
section := 4
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar_1", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: candidates, SectionIndex: &section, Exchange: &structuredoutput.Exchange{}, Interactions: llm.NewDiagnosticsWriter(t.TempDir())}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar_1", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{rejectUntilRepair()}, SectionEnriched: candidates, SectionIndex: &section, Exchange: &structuredoutput.Exchange{}, Interactions: llm.NewDiagnosticsWriter(t.TempDir())}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
@@ -355,7 +366,7 @@ func TestRepairArtifactFailureDoesNotChangeOutcome(t *testing.T) {
}
sink := llm.NewDiagnosticsWriter(path)
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}, Interactions: sink}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}, Interactions: sink}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
@@ -382,8 +393,8 @@ func TestNoEffectAndUnavailableOnlyDoNotStartRepair(t *testing.T) {
}
// rejectUntilRepair supplies a semantic rejection at the external validator seam.
func rejectUntilRepair(preserved ...int) contracts.Validator {
return fakeValidator{name: "editorial_review", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
func rejectUntilRepair(preserved ...int) validators.Validator {
return fakeValidator{name: "editorial_review", validateF: func(req validators.Request) (validators.Result, error) {
var decisions []validators.Decision
for _, p := range req.CandidateProposal {
approved := req.RepairAttempt != nil

View File

@@ -69,13 +69,14 @@ type WorkUnitOutcome struct {
}
type ValidatorDecisionRecord struct {
ValidatorName string `json:"validator_name"`
ProposalIndex int `json:"proposal_index"`
Approved bool `json:"approved"`
Disposition validators.DecisionDisposition `json:"disposition"`
ReasonCode string `json:"reason_code"`
Message string `json:"message"`
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
ExecutionClass validatormetadata.ExecutionClass `json:"-"`
ValidatorName string `json:"validator_name"`
ProposalIndex int `json:"proposal_index"`
Approved bool `json:"approved"`
Disposition validators.DecisionDisposition `json:"disposition"`
ReasonCode string `json:"reason_code"`
Message string `json:"message"`
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
}
type ValidatorRejectedChange struct {
@@ -650,7 +651,7 @@ type validateSectionCandidatesInput struct {
Policy proposals.ReplacementPolicy
WorkingTranscript *schema.Transcript
SectionIndex *int
Validators []contracts.Validator
Validators []validators.Validator
SectionEnriched []proposals.EnrichedCorrectionProposal
ValidationLLMClient contracts.StructuredLLMClient
Interactions structuredoutput.InteractionSink
@@ -673,7 +674,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
eligible := append([]proposals.EnrichedCorrectionProposal(nil), input.SectionEnriched...)
feedback := input.Spec.Settings.LLM.CorrectionRepairAttempts > 0
var blocked, noEffect map[int]bool
evidence := contracts.ValidationRequest{WorkingTranscript: input.WorkingTranscript, Section: input.Section, ReplacementPolicy: input.Policy, IndependentEvidence: feedback}
evidence := validators.Request{WorkingTranscript: input.WorkingTranscript, Section: input.Section, ReplacementPolicy: input.Policy, IndependentEvidence: feedback}
if feedback {
blocked, noEffect = make(map[int]bool), make(map[int]bool)
evidence.Previews = make(map[int]proposals.SegmentPreviewResult)
@@ -686,11 +687,12 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
if err := ctx.Err(); err != nil {
return validateSectionCandidatesResult{decisions: decisions, rejected: rejected, operations: operations, filtered: filtered}, err
}
class := validatormetadata.ClassOf(validator)
submitted := eligible
if feedback {
submitted = feedbackCandidates(input, validator, noEffect, evidence)
submitted = feedbackCandidates(input, validator.Name(), class == validatormetadata.ExecutionClassLLMBacked, noEffect, evidence)
}
vResult, err := validator.Validate(ctx, contracts.ValidationRequest{
vResult, err := validator.Validate(ctx, validators.Request{
StageIndex: &input.Spec.Index, StageID: input.Spec.ID,
WorkingTranscript: input.WorkingTranscript,
Section: evidence.Section, Previews: evidence.Previews,
@@ -728,17 +730,20 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
byIndex[p.ProposalIndex] = p
}
for _, d := range vResult.Decisions {
if feedback && validator.Name() == "original_text_presence" && d.Approved {
preview := evidence.Preview(byIndex[d.ProposalIndex])
if preview.SkipReason == proposals.SkipReasonAmbiguousOriginal {
d.Approved = false
d.Disposition = validators.DispositionRejected
d.ReasonCode = string(proposals.SkipReasonAmbiguousOriginal)
d.Message = "original_text matches multiple locations in the target segment"
}
dispositions := make([]validators.DecisionDisposition, len(vResult.Decisions))
for i, decision := range vResult.Decisions {
disposition, contractErr := decision.EffectiveDisposition()
if contractErr != nil {
return validateSectionCandidatesResult{
decisions: decisions, rejected: rejected, operations: operations, filtered: filtered,
}, fmt.Errorf("validator %q returned inconsistent decisions: %w", validator.Name(), errors.Join(err, contractErr))
}
if feedback && !d.Approved {
dispositions[i] = disposition
}
for i, d := range vResult.Decisions {
disposition := dispositions[i]
approved := disposition == validators.DispositionApproved
if feedback && !approved {
blocked[d.ProposalIndex] = true
if d.ReasonCode == validators.ReasonNoEffect {
noEffect[d.ProposalIndex] = true
@@ -746,18 +751,19 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
}
decisions = append(decisions, ValidatorDecisionRecord{
ValidatorName: validator.Name(),
ExecutionClass: class,
ProposalIndex: d.ProposalIndex,
Approved: d.Approved,
Disposition: d.Disposition,
Approved: approved,
Disposition: disposition,
ReasonCode: d.ReasonCode,
Message: d.Message,
DiagnosticArtifactPath: d.DiagnosticArtifactPath,
})
if d.Approved {
if approved {
nextEligible = append(nextEligible, byIndex[d.ProposalIndex])
continue
}
if d.Disposition != validators.DispositionUnavailable {
if disposition != validators.DispositionUnavailable {
if reason, ok := validators.DefectiveProposalReason(d.ReasonCode); ok {
filtered = append(filtered, outcomes.FilteredProposal{ProposalIndex: d.ProposalIndex, Reason: reason, Count: 1})
}
@@ -771,7 +777,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
TargetSegmentID: p.TargetSegmentID,
OriginalText: p.OriginalText,
CorrectedText: p.CorrectedText,
Disposition: d.Disposition,
Disposition: disposition,
ReasonCode: d.ReasonCode,
Message: d.Message,
})
@@ -882,11 +888,13 @@ func attributeApplyFiltering(sections []chunking.Section, approved []proposals.E
}
}
func reorderValidatorsForPipeline(in []contracts.Validator) ([]contracts.Validator, map[string]int) {
deterministic := make([]contracts.Validator, 0, len(in))
llmBacked := make([]contracts.Validator, 0, len(in))
func reorderValidatorsForPipeline(in []validators.Validator) ([]validators.Validator, map[string]int) {
deterministic := make([]validators.Validator, 0, len(in))
llmBacked := make([]validators.Validator, 0, len(in))
for _, validator := range in {
if validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked {
class := validatormetadata.ClassOf(validator)
validator = validatormetadata.Wrap(validator, class)
if class == validatormetadata.ExecutionClassLLMBacked {
llmBacked = append(llmBacked, validator)
continue
}

View File

@@ -43,7 +43,7 @@ func (f fakeFactory) ModuleForSpec(spec contracts.ModuleRunSpec) (contracts.Tran
type fakeModule struct {
key string
policy proposals.ReplacementPolicy
validators []contracts.Validator
validators []validators.Validator
proposeF func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error)
}
@@ -52,6 +52,33 @@ type outcomeModule struct {
proposeResult func(contracts.ProposalRequest) (contracts.ProposalResult, error)
}
func TestRunnerStopsOnContradictoryValidatorDecision(t *testing.T) {
candidate := proposals.EnrichedCorrectionProposal{
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"},
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 7},
}
contractFailure := errors.New("validator also returned a runtime error")
for _, cause := range []error{nil, contractFailure} {
contradictory := fakeValidator{name: "invalid", validateF: func(validators.Request) (validators.Result, error) {
return validators.Result{Decisions: []validators.Decision{{
ProposalIndex: 7, Approved: true, Disposition: validators.DispositionUnavailable,
}}}, cause
}}
later := fakeValidator{name: "later", validateF: func(validators.Request) (validators.Result, error) {
t.Fatal("candidate advanced past a contradictory decision")
return validators.Result{}, nil
}}
result, err := validateSectionCandidates(context.Background(), validateSectionCandidatesInput{
Validators: []validators.Validator{validators.ProposalShapeValidator{}, contradictory, later},
SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate},
})
if err == nil || !strings.Contains(err.Error(), "contradictory") || (cause != nil && !errors.Is(err, cause)) ||
len(result.approved) != 0 || len(result.decisions) != 1 || result.decisions[0].ValidatorName != "proposal_shape" {
t.Fatalf("contradictory result = %+v (%v)", result, err)
}
}
}
func TestRunnerRecoverableProposalFailureAllowsLaterModule(t *testing.T) {
failed := outcomeModule{fakeModule: fakeModule{key: "first", policy: proposals.ReplacementPolicyRequireUnique}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
return contracts.ProposalResult{Operations: []outcomes.Operation{{Reference: outcomes.Reference{ModuleKey: "first", ModuleInstance: "first", Kind: outcomes.OperationProposal}, FailureReason: outcomes.ReasonTransportFailed}}}, nil
@@ -74,7 +101,7 @@ func TestRunnerRecoverableProposalFailureAllowsLaterModule(t *testing.T) {
func TestRunnerOnlyFullyValidatedCandidatesApplyAndRecoveriesMerge(t *testing.T) {
secondSaw := make([]int, 0)
partial := fakeValidator{name: "partial", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
partial := fakeValidator{name: "partial", validateF: func(req validators.Request) (validators.Result, error) {
if len(req.CandidateProposal) != 2 {
t.Fatalf("expected two candidates after shape filtering, got %d", len(req.CandidateProposal))
}
@@ -86,13 +113,13 @@ func TestRunnerOnlyFullyValidatedCandidatesApplyAndRecoveriesMerge(t *testing.T)
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation}, FailureReason: outcomes.ReasonProviderFailed},
}}, nil
}}
last := fakeValidator{name: "last", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
last := fakeValidator{name: "last", validateF: func(req validators.Request) (validators.Result, error) {
for _, candidate := range req.CandidateProposal {
secondSaw = append(secondSaw, candidate.ProposalIndex)
}
return validators.Result{ValidatorName: "last", Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: true, Disposition: validators.DispositionApproved}}}, nil
}}
module := outcomeModule{fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{validators.ProposalShapeValidator{}, partial, last}}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
module := outcomeModule{fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{validators.ProposalShapeValidator{}, partial, last}}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
return contracts.ProposalResult{Proposals: []proposals.CorrectionProposal{
{TargetSegmentID: 0, OriginalText: "bad", CorrectedText: "x"},
{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"},
@@ -120,10 +147,10 @@ func TestRunnerOnlyFullyValidatedCandidatesApplyAndRecoveriesMerge(t *testing.T)
func TestFatalValidatorNeverReturnsUncheckedCandidates(t *testing.T) {
origin := errors.New("validator setup failed")
proposal := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 7}}
broken := fakeValidator{name: "broken", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
broken := fakeValidator{name: "broken", validateF: func(req validators.Request) (validators.Result, error) {
return validators.Result{Operations: []outcomes.Operation{{Reference: outcomes.Reference{Kind: outcomes.OperationValidation}, FailureReason: outcomes.ReasonSetupFailure}}}, origin
}}
result, err := validateSectionCandidates(context.Background(), validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "m", InstanceName: "m"}, Validators: []contracts.Validator{broken}, SectionEnriched: []proposals.EnrichedCorrectionProposal{proposal}})
result, err := validateSectionCandidates(context.Background(), validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "m", InstanceName: "m"}, Validators: []validators.Validator{broken}, SectionEnriched: []proposals.EnrichedCorrectionProposal{proposal}})
if !errors.Is(err, origin) || len(result.approved) != 0 || len(result.operations) != 1 {
t.Fatalf("fatal result must retain facts without approving candidates: result=%+v err=%v", result, err)
}
@@ -131,7 +158,7 @@ func TestFatalValidatorNeverReturnsUncheckedCandidates(t *testing.T) {
func TestCandidateCaptureIsExplicitAndIncludesFilteredProposals(t *testing.T) {
module := fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique,
validators: []contracts.Validator{validators.ProposalShapeValidator{}},
validators: []validators.Validator{validators.ProposalShapeValidator{}},
proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 0, OriginalText: "bad", CorrectedText: "good"}}, nil
}}
@@ -160,7 +187,7 @@ func (m outcomeModule) Propose(_ context.Context, req contracts.ProposalRequest)
func (m fakeModule) Key() string { return m.key }
func (m fakeModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy }
func (m fakeModule) Validators() []contracts.Validator { return m.validators }
func (m fakeModule) Validators() []validators.Validator { return m.validators }
func (m fakeModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
if m.proposeF == nil {
return contracts.ProposalResult{}, nil
@@ -171,11 +198,11 @@ func (m fakeModule) Propose(ctx context.Context, req contracts.ProposalRequest)
type fakeValidator struct {
name string
validateF func(req contracts.ValidationRequest) (validators.Result, error)
validateF func(req validators.Request) (validators.Result, error)
}
func (v fakeValidator) Name() string { return v.name }
func (v fakeValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (validators.Result, error) {
func (v fakeValidator) Validate(ctx context.Context, req validators.Request) (validators.Result, error) {
_ = ctx
return v.validateF(req)
}
@@ -191,19 +218,19 @@ func (v classifiedFakeValidator) ExecutionClass() validatormetadata.ExecutionCla
func TestReorderValidatorsDeterministicBeforeLLMBacked(t *testing.T) {
llm := classifiedFakeValidator{
fakeValidator: fakeValidator{name: "llm", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
fakeValidator: fakeValidator{name: "llm", validateF: func(req validators.Request) (validators.Result, error) {
return validators.Result{ValidatorName: "llm"}, nil
}},
class: validatormetadata.ExecutionClassLLMBacked,
}
deterministic := classifiedFakeValidator{
fakeValidator: fakeValidator{name: "deterministic", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
fakeValidator: fakeValidator{name: "deterministic", validateF: func(req validators.Request) (validators.Result, error) {
return validators.Result{ValidatorName: "deterministic"}, nil
}},
class: validatormetadata.ExecutionClassDeterministic,
}
ordered, _ := reorderValidatorsForPipeline([]contracts.Validator{llm, deterministic})
ordered, _ := reorderValidatorsForPipeline([]validators.Validator{llm, deterministic})
if len(ordered) != 2 {
t.Fatalf("expected 2 validators, got %d", len(ordered))
}
@@ -213,17 +240,17 @@ func TestReorderValidatorsDeterministicBeforeLLMBacked(t *testing.T) {
}
func TestReorderValidatorsDefaultsUnclassifiedToDeterministic(t *testing.T) {
unclassified := fakeValidator{name: "plain", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
unclassified := fakeValidator{name: "plain", validateF: func(req validators.Request) (validators.Result, error) {
return validators.Result{ValidatorName: "plain"}, nil
}}
llm := classifiedFakeValidator{
fakeValidator: fakeValidator{name: "llm", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
fakeValidator: fakeValidator{name: "llm", validateF: func(req validators.Request) (validators.Result, error) {
return validators.Result{ValidatorName: "llm"}, nil
}},
class: validatormetadata.ExecutionClassLLMBacked,
}
ordered, _ := reorderValidatorsForPipeline([]contracts.Validator{llm, unclassified})
ordered, _ := reorderValidatorsForPipeline([]validators.Validator{llm, unclassified})
if len(ordered) != 2 {
t.Fatalf("expected 2 validators, got %d", len(ordered))
}
@@ -553,7 +580,7 @@ func TestRunnerValidationStartsBeforeAllSectionProposalsComplete(t *testing.T) {
validator := fakeValidator{
name: "capture",
validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
validateF: func(req validators.Request) (validators.Result, error) {
if len(req.CandidateProposal) == 1 && req.CandidateProposal[0].TargetSegmentID == 1 {
select {
case validatorStarted <- struct{}{}:
@@ -577,7 +604,7 @@ func TestRunnerValidationStartsBeforeAllSectionProposalsComplete(t *testing.T) {
"m": fakeModule{
key: "m",
policy: proposals.ReplacementPolicyRequireUnique,
validators: []contracts.Validator{validator},
validators: []validators.Validator{validator},
proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
seg := req.WorkingTranscript.Segments[0]
if req.Section != nil && req.Section.Index == 1 {
@@ -630,7 +657,7 @@ func (c *captureContractStructuredClient) CompleteStructured(ctx context.Context
if target, ok := out.(*validators.LLMValidationResponse); ok {
*target = validators.LLMValidationResponse{
Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
{CorrectionIndex: "1", Approved: true, Reason: "ok"},
},
}
}
@@ -662,7 +689,7 @@ func (c *stageAwareStructuredClient) CompleteStructured(ctx context.Context, req
target := out.(*validators.LLMValidationResponse)
*target = validators.LLMValidationResponse{
Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
{CorrectionIndex: "1", Approved: true, Reason: "ok"},
},
}
return contracts.StructuredCompletionResponse{}, nil
@@ -734,7 +761,7 @@ func TestRunnerRepeatedModuleInstanceNames(t *testing.T) {
func TestRunnerValidatorApprovedProposalApplied(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}}
allowAll := fakeValidator{name: "allow", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
allowAll := fakeValidator{name: "allow", validateF: func(req validators.Request) (validators.Result, error) {
decisions := make([]validators.Decision, len(req.CandidateProposal))
for i, p := range req.CandidateProposal {
decisions[i] = validators.Decision{ProposalIndex: p.ProposalIndex, Approved: true, ReasonCode: validators.ReasonApproved, Message: "approved"}
@@ -745,7 +772,7 @@ func TestRunnerValidatorApprovedProposalApplied(t *testing.T) {
"m": fakeModule{
key: "m",
policy: proposals.ReplacementPolicyRequireUnique,
validators: []contracts.Validator{allowAll},
validators: []validators.Validator{allowAll},
proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil
},
@@ -766,7 +793,7 @@ func TestRunnerValidatorApprovedProposalApplied(t *testing.T) {
func TestRunnerValidatorRejectedProposalNotApplied(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}}
rejectAll := fakeValidator{name: "reject", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
rejectAll := fakeValidator{name: "reject", validateF: func(req validators.Request) (validators.Result, error) {
decisions := make([]validators.Decision, len(req.CandidateProposal))
for i, p := range req.CandidateProposal {
decisions[i] = validators.Decision{ProposalIndex: p.ProposalIndex, Approved: false, ReasonCode: validators.ReasonNoEffect, Message: "rejected"}
@@ -774,7 +801,7 @@ func TestRunnerValidatorRejectedProposalNotApplied(t *testing.T) {
return validators.Result{ValidatorName: "reject", Decisions: decisions}, nil
}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{rejectAll}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{rejectAll}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil
}},
}})
@@ -792,7 +819,7 @@ func TestRunnerValidatorRejectedProposalNotApplied(t *testing.T) {
func TestRunnerMultipleValidatorsRunInOrderAndFilterSurvivors(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "one two"}}}
first := fakeValidator{name: "first", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
first := fakeValidator{name: "first", validateF: func(req validators.Request) (validators.Result, error) {
if len(req.CandidateProposal) != 2 {
t.Fatalf("expected first validator to see 2 candidates, got %d", len(req.CandidateProposal))
}
@@ -804,14 +831,14 @@ func TestRunnerMultipleValidatorsRunInOrderAndFilterSurvivors(t *testing.T) {
},
}, nil
}}
second := fakeValidator{name: "second", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
second := fakeValidator{name: "second", validateF: func(req validators.Request) (validators.Result, error) {
if len(req.CandidateProposal) != 1 {
t.Fatalf("expected second validator to see only survivors, got %d", len(req.CandidateProposal))
}
return validators.Result{ValidatorName: "second", Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: true, ReasonCode: validators.ReasonApproved, Message: "ok"}}}, nil
}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{first, second}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{first, second}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{
{TargetSegmentID: 1, OriginalText: "one", CorrectedText: "ONE"},
{TargetSegmentID: 1, OriginalText: "two", CorrectedText: "TWO"},
@@ -833,11 +860,11 @@ func TestRunnerValidatorCardinalityErrorStopsPipelineWithPartialProgress(t *test
good := fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil
}}
badValidator := fakeValidator{name: "bad", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
badValidator := fakeValidator{name: "bad", validateF: func(req validators.Request) (validators.Result, error) {
// Missing one decision triggers cardinality error.
return validators.Result{ValidatorName: "bad", Decisions: nil}, nil
}}
bad := fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{badValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
bad := fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{badValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}}, nil
}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"m1": good, "m2": bad}})
@@ -856,11 +883,11 @@ func TestRunnerValidatorCardinalityErrorStopsPipelineWithPartialProgress(t *test
func TestRunnerApplicationSkipAfterValidatorApprovalReported(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "word word"}}}
allow := fakeValidator{name: "allow", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
allow := fakeValidator{name: "allow", validateF: func(req validators.Request) (validators.Result, error) {
return validators.Result{ValidatorName: "allow", Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: true, ReasonCode: validators.ReasonApproved, Message: "ok"}}}, nil
}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{allow}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{allow}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "word", CorrectedText: "term"}}, nil
}},
}})
@@ -880,7 +907,7 @@ func TestRunnerApplyConflictAfterTwoInitiallyValidDeletions(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "a b"}}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": outcomeModule{
fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NonEmptyResultingSegmentValidator{}}},
fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NonEmptyResultingSegmentValidator{}}},
proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
return contracts.ProposalResult{
Proposals: []proposals.CorrectionProposal{
@@ -945,14 +972,14 @@ func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contr
}
func TestRunnerLLMValidatorApprovalApplied(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}}}
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}}}
llmValidator, err := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
if err != nil {
t.Fatalf("NewLLMBackedValidator: %v", err)
}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil
}},
}})
@@ -971,10 +998,10 @@ func TestRunnerLLMValidatorApprovalApplied(t *testing.T) {
}
func TestRunnerLLMValidatorRejectionPreventsApplication(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: false, Reason: "reject"}}}}}
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: false, Reason: "reject"}}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil
}},
}})
@@ -996,13 +1023,13 @@ func TestRunnerLLMValidatorRejectionPreventsApplication(t *testing.T) {
}
func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Reason: "bad index"}}}}}
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "99", Approved: true, Reason: "bad index"}}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m1": fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil
}},
"m2": fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
"m2": fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}}, nil
}},
}})
@@ -1028,13 +1055,13 @@ func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress(
func TestRunnerReconciledSurvivorsCompleteLaterValidation(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
{CorrectionIndex: 3, Approved: true, Reason: "ok"},
{CorrectionIndex: 99, Approved: true, Reason: "unexpected"},
{CorrectionIndex: "1", Approved: true, Reason: "ok"},
{CorrectionIndex: "3", Approved: true, Reason: "ok"},
{CorrectionIndex: "99", Approved: true, Reason: "unexpected"},
}}}}
first, _ := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
var laterIndices []int
later := fakeValidator{name: "later_review", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
later := fakeValidator{name: "later_review", validateF: func(req validators.Request) (validators.Result, error) {
decisions := make([]validators.Decision, 0, len(req.CandidateProposal))
for _, candidate := range req.CandidateProposal {
laterIndices = append(laterIndices, candidate.ProposalIndex)
@@ -1047,7 +1074,7 @@ func TestRunnerReconciledSurvivorsCompleteLaterValidation(t *testing.T) {
}
return validators.Result{ValidatorName: "later_review", Decisions: decisions}, nil
}}
module := outcomeModule{fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{first, validatormetadata.Wrap(later, validatormetadata.ExecutionClassLLMBacked)}}, proposeResult: func(contracts.ProposalRequest) (contracts.ProposalResult, error) {
module := outcomeModule{fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{first, validatormetadata.Wrap(later, validatormetadata.ExecutionClassLLMBacked)}}, proposeResult: func(contracts.ProposalRequest) (contracts.ProposalResult, error) {
return contracts.ProposalResult{Proposals: []proposals.CorrectionProposal{
{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"},
{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"},
@@ -1077,7 +1104,7 @@ func TestRunnerLLMValidatorMissingDecisionRejectsBatch(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil
}},
}})
@@ -1094,12 +1121,12 @@ func TestRunnerLLMValidatorMissingDecisionRejectsBatch(t *testing.T) {
func TestRunnerLLMValidatorDuplicateDecisionRejectsBatch(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
{CorrectionIndex: 1, Approved: false, Reason: "dup"},
{CorrectionIndex: "1", Approved: true, Reason: "ok"},
{CorrectionIndex: "1", Approved: false, Reason: "dup"},
}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil
}},
}})
@@ -1120,8 +1147,8 @@ func (lenEstimator) EstimateTokens(text string) int { return len(text) }
func TestRunnerLLMValidatorBatching(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}},
}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
llmValidator.SetTokenEstimator(lenEstimator{})
@@ -1129,7 +1156,7 @@ func TestRunnerLLMValidatorBatching(t *testing.T) {
cfg := runnerTestConfig()
cfg.Validation.MaxPromptTokens = 260
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyReplaceAll, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyReplaceAll, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{
{TargetSegmentID: 1, OriginalText: "alpha", CorrectedText: strings.Repeat("B", 40)},
{TargetSegmentID: 1, OriginalText: "gamma", CorrectedText: strings.Repeat("D", 40)},
@@ -1154,12 +1181,12 @@ func TestRunnerLLMValidatorBatching(t *testing.T) {
}
func TestRunnerLLMValidatorDiagnosticsWritten(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "supported by context"}}}}}
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "supported by context"}}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
cfg := runnerTestConfig()
diagDir := t.TempDir()
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil
}},
}})
@@ -1208,7 +1235,7 @@ type proposalGenerationModule struct {
func (m proposalGenerationModule) Key() string { return m.key }
func (m proposalGenerationModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy }
func (m proposalGenerationModule) Validators() []contracts.Validator { return nil }
func (m proposalGenerationModule) Validators() []validators.Validator { return nil }
func (m proposalGenerationModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
result, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
ModuleKey: req.RunSpec.ModuleKey,
@@ -1294,10 +1321,10 @@ func TestFatalPartialDecisionsRejectUnknownAndDuplicateIdentity(t *testing.T) {
{{ProposalIndex: 8, Approved: true}},
{{ProposalIndex: 7, Approved: true}, {ProposalIndex: 7, Approved: true}},
} {
v := fakeValidator{name: "broken", validateF: func(contracts.ValidationRequest) (validators.Result, error) {
v := fakeValidator{name: "broken", validateF: func(validators.Request) (validators.Result, error) {
return validators.Result{Decisions: decisions}, origin
}}
result, err := validateSectionCandidates(context.Background(), validateSectionCandidatesInput{Validators: []contracts.Validator{v}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}})
result, err := validateSectionCandidates(context.Background(), validateSectionCandidatesInput{Validators: []validators.Validator{v}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}})
if !errors.Is(err, origin) || len(result.decisions) != 0 || len(result.approved) != 0 {
t.Fatalf("invalid partial evidence accepted or origin lost: %+v %v", result, err)
}

View File

@@ -37,6 +37,7 @@ func (v OriginalTextPresenceValidator) Name() string { return "original_text_pre
func (v OriginalTextPresenceValidator) Validate(_ context.Context, req Request) (Result, error) {
section := req.sectionLookup()
req.Section = section
decisions := make([]Decision, 0, len(req.CandidateProposal))
for _, c := range req.CandidateProposal {
@@ -49,6 +50,10 @@ func (v OriginalTextPresenceValidator) Validate(_ context.Context, req Request)
decisions = append(decisions, rejection(c.ProposalIndex, ReasonMissingOriginalText, "original_text was not found in current segment text"))
continue
}
if req.IndependentEvidence && req.Preview(c).SkipReason == proposals.SkipReasonAmbiguousOriginal {
decisions = append(decisions, rejection(c.ProposalIndex, string(proposals.SkipReasonAmbiguousOriginal), "original_text matches multiple locations in the target segment"))
continue
}
decisions = append(decisions, approval(c.ProposalIndex))
}
if err := EnforceDecisionCardinality(req.CandidateProposal, decisions); err != nil {

View File

@@ -25,15 +25,15 @@ func TestValidationContextCarriesAdjacentEvidenceAndRelevantDefinitions(t *testi
{ID: "lyra", Name: "Lyra", Description: "The host"},
{ID: "unrelated", Name: "Unrelated", Description: "Not mentioned"},
}
payload, immediate := BuildLLMValidationRequest("spoken_form_plausibility", LLMValidatorTypeSpokenFormPlausibility, req)
if len(immediate) != 0 || len(payload.Items) != 1 {
payload, immediate := buildValidationItems(req)
if len(immediate) != 0 || len(payload) != 1 {
t.Fatalf("payload: %+v immediate: %+v", payload, immediate)
}
item := payload.Items[0]
item := payload[0]
if len(item.NeighboringSegments) != 2 || item.NeighboringSegments[0].Position != "preceding" || item.NeighboringSegments[1].Position != "following" || len(item.GlossaryEntries) != 2 || item.GlossaryEntries[0].Name != "Jesters" || item.GlossaryEntries[1].Name != "Lyra" {
t.Fatalf("context: %+v", item)
}
variables, err := BuildValidationVariables(payload.Items, "")
variables, err := BuildValidationVariables(payload, "")
if err != nil {
t.Fatal(err)
}
@@ -41,7 +41,7 @@ func TestValidationContextCarriesAdjacentEvidenceAndRelevantDefinitions(t *testi
if err := json.Unmarshal([]byte(variables["PayloadJSON"]), &rendered); err != nil {
t.Fatal(err)
}
if strings.Contains(variables["PayloadJSON"], `"segment_id":`) || rendered[0].CorrectionIndex != 1 || payload.Items[0].CorrectionIndex != 7 {
if strings.Contains(variables["PayloadJSON"], `"segment_id":`) || rendered[0].CorrectionIndex != 1 || payload[0].CorrectionIndex != 7 {
t.Fatalf("leaked internal identity or mutated request: %s", variables["PayloadJSON"])
}
if rendered[0].Speaker != "Guest" || len(rendered[0].NeighboringSegments) != 2 || rendered[0].NeighboringSegments[0].Speaker != "Host" || rendered[0].GlossaryEntries[0].Description != "A traveling troupe" {
@@ -51,15 +51,15 @@ func TestValidationContextCarriesAdjacentEvidenceAndRelevantDefinitions(t *testi
req.WorkingTranscript.Segments = req.WorkingTranscript.Segments[1:2]
req.WorkingTranscript.Segments[0].Speaker = ""
req.Glossary = nil
isolated, _ := BuildLLMValidationRequest("editorial_review", LLMValidatorTypeEditorialReview, req)
if isolated.Items[0].Speaker != "" || len(isolated.Items[0].NeighboringSegments) != 0 || len(isolated.Items[0].GlossaryEntries) != 0 {
isolated, _ := buildValidationItems(req)
if isolated[0].Speaker != "" || len(isolated[0].NeighboringSegments) != 0 || len(isolated[0].GlossaryEntries) != 0 {
t.Fatalf("unexpected external context: %+v", isolated)
}
}
func TestValidationContextFitsBudgetWithoutDiscardingCandidate(t *testing.T) {
estimator := chunking.NewSimpleTokenEstimator()
base := LLMValidationItem{CorrectionIndex: 7, SegmentID: 1, OriginalText: "teh", CorrectedText: "the", OriginalSegmentText: "teh cat", CorrectedSegmentText: "the cat"}
base := LLMValidationItem{CorrectionIndex: 7, OriginalText: "teh", CorrectedText: "the", OriginalSegmentText: "teh cat", CorrectedSegmentText: "the cat"}
required, err := estimateBatchTokens(estimator, []LLMValidationItem{base})
if err != nil {
t.Fatal(err)
@@ -89,12 +89,12 @@ func TestValidationContextSelectsGlossaryForEachReplacement(t *testing.T) {
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(7, "gestures", "Jesters"), mk(11, "gestures", "Actors")})
req.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Those gestures are back."}}}
req.Glossary = []glossary.Entry{{ID: "actors", Name: "Actors"}, {ID: "jesters", Name: "Jesters"}}
payload, immediate := BuildLLMValidationRequest("editorial_review", LLMValidatorTypeEditorialReview, req)
if len(immediate) != 0 || len(payload.Items) != 2 {
payload, immediate := buildValidationItems(req)
if len(immediate) != 0 || len(payload) != 2 {
t.Fatalf("payload: %+v immediate: %+v", payload, immediate)
}
for i, name := range []string{"Jesters", "Actors"} {
entries := payload.Items[i].GlossaryEntries
entries := payload[i].GlossaryEntries
if len(entries) != 1 || entries[0].Name != name {
t.Fatalf("candidate %d context: %+v", i, entries)
}

View File

@@ -15,6 +15,7 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
ck "go.maximumdirect.net/configkit"
)
@@ -50,28 +51,7 @@ func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) {
}
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
})
dir := t.TempDir()
main := filepath.Join(dir, "config.yml")
if err := os.WriteFile(main, []byte("version: 3\n"), 0600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte("profiles:\n - {id: base, endpoint: 'https://example.invalid/v1', model: local-model, api_key_required: false}\n"), 0600); err != nil {
t.Fatal(err)
}
loaded, err := config.Load(ck.Input{Source: ck.File(main)}, "")
if err != nil {
t.Fatal(err)
}
runtime, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
if err != nil {
t.Fatal(err)
}
settings := loaded.Config.LLM
settings.TransientRetries, settings.RepairAttempts = 0, 1
client, err := runtime.NewRoleClient("validation", "base", settings, llm.CallHooks{})
if err != nil {
t.Fatal(err)
}
client := testValidationClient(t, transport)
validator, err := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
if err != nil {
t.Fatal(err)
@@ -91,3 +71,98 @@ func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) {
})
}
}
// The transport supplies provider responses; PromptKit performs structured
// validation and Audita decodes the same response through the role client.
func testValidationClient(t *testing.T, transport http.RoundTripper) structuredoutput.Client {
t.Helper()
dir := t.TempDir()
main := filepath.Join(dir, "config.yml")
if err := os.WriteFile(main, []byte("version: 3\n"), 0600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte("profiles:\n - {id: base, endpoint: 'https://example.invalid/v1', model: local-model, api_key_required: false}\n"), 0600); err != nil {
t.Fatal(err)
}
loaded, err := config.Load(ck.Input{Source: ck.File(main)}, "")
if err != nil {
t.Fatal(err)
}
runtime, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
if err != nil {
t.Fatal(err)
}
settings := loaded.Config.LLM
settings.TransientRetries, settings.RepairAttempts = 0, 1
client, err := runtime.NewRoleClient("validation", "base", settings, llm.CallHooks{})
if err != nil {
t.Fatal(err)
}
return client
}
func TestValidationDecodesExactReferencesBeforeReconciliation(t *testing.T) {
for _, tc := range []struct {
name, response string
indices []int
want []DecisionDisposition
reasons []string
failure outcomes.ReasonCode
}{
{
name: "usable decision beside extra",
response: `{"validations":[{"correction_index":1.00e0,"approved":true,"reason":"accept"},{"correction_index":9223372036854775808,"approved":true,"reason":"extra"}]}`,
indices: []int{7},
want: []DecisionDisposition{DispositionApproved},
reasons: []string{ReasonApproved},
},
{
name: "usable siblings beside duplicate and omission",
response: `{"validations":[{"correction_index":1e0,"approved":true,"reason":"accept"},{"correction_index":2,"approved":true,"reason":"duplicate"},{"correction_index":20e-1,"approved":false,"reason":"duplicate"},{"correction_index":4.000,"approved":false,"reason":"reject"},{"correction_index":9223372036854775808,"approved":true,"reason":"extra"}]}`,
indices: []int{7, 11, 14, 23},
want: []DecisionDisposition{DispositionApproved, DispositionUnavailable, DispositionUnavailable, DispositionRejected},
reasons: []string{ReasonApproved, ReasonDuplicateValidationDecision, ReasonMissingValidationDecision, "llm_rejected"},
failure: outcomes.ReasonValidationDecisionsInvalid,
},
} {
t.Run(tc.name, func(t *testing.T) {
calls := 0
client := testValidationClient(t, validationTransport(func(req *http.Request) (*http.Response, error) {
calls++
content, _ := json.Marshal(tc.response)
body := fmt.Sprintf(`{"choices":[{"message":{"content":%s}}]}`, content)
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: req}, nil
}))
var candidates []proposals.EnrichedCorrectionProposal
for _, index := range tc.indices {
candidates = append(candidates, mk(index, "gestures", "Jesters"))
}
validator, err := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
if err != nil {
t.Fatal(err)
}
sink := &captureValidationDiagnosticsWriter{}
req := makeReq(candidates)
req.LLMClient = client
req.Interactions = sink
result, err := validator.Validate(context.Background(), req)
if err != nil || calls != 1 || len(result.Decisions) != len(tc.want) || len(result.Operations) != 1 {
t.Fatalf("result=%+v calls=%d error=%v", result, calls, err)
}
for i, decision := range result.Decisions {
if decision.ProposalIndex != tc.indices[i] || decision.Disposition != tc.want[i] || decision.ReasonCode != tc.reasons[i] {
t.Fatalf("decision %d: %+v", i, decision)
}
}
operation := result.Operations[0]
if operation.FailureReason != tc.failure || (tc.failure == "" && (len(operation.RecoveryReasons) != 1 || operation.RecoveryReasons[0] != outcomes.RecoveryValidationDecisionsFiltered)) {
t.Fatalf("operation: %+v", operation)
}
payload := sink.lastResponsePayload.(map[string]any)
diagnostics := payload["reconciliation"].(validationReconciliation)
if len(diagnostics.IgnoredIndices) != 1 || diagnostics.IgnoredIndices[0] != "9223372036854775808" || payload["final_content"] != tc.response {
t.Fatalf("reference evidence changed: %+v", payload)
}
})
}
}

View File

@@ -1,6 +1,8 @@
package validators
import (
"encoding/json"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
)
@@ -21,7 +23,6 @@ type StructuredLLMClient = structuredoutput.Client
// BuildValidationVariables substitutes a one-based reference for each batch.
type LLMValidationItem struct {
CorrectionIndex int `json:"correction_index"`
SegmentID int `json:"-"`
Speaker string `json:"speaker"`
OriginalText string `json:"original_text"`
CorrectedText string `json:"corrected_text"`
@@ -32,21 +33,11 @@ type LLMValidationItem struct {
GlossaryEntries []glossary.Entry `json:"glossary_entries,omitempty"`
}
// LLMValidationRequest is the canonical request model for LLM-backed validators.
type LLMValidationRequest struct {
ValidatorName string `json:"validator_name"`
ValidatorType LLMValidatorType `json:"validator_type"`
ModuleKey string `json:"module_key"`
ModuleInstance string `json:"module_instance"`
ReplacementPolicy string `json:"replacement_policy"`
Items []LLMValidationItem `json:"corrections"`
}
// LLMValidationDecision contains a request-local correction_index, not a proposal index.
type LLMValidationDecision struct {
CorrectionIndex int `json:"correction_index"`
Approved bool `json:"approved"`
Reason string `json:"reason"`
CorrectionIndex json.Number `json:"correction_index"`
Approved bool `json:"approved"`
Reason string `json:"reason"`
}
type LLMValidationResponse struct {

View File

@@ -48,9 +48,9 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
}
failed := &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonTransportFailed, Message: "model transport failed"}
client := &scriptedValidationClient{replies: []validationReply{
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}},
{completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 1}}, err: failed},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}},
}}
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
validator.SetTokenEstimator(perItemEstimator{})
@@ -73,8 +73,8 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
survivors := []proposals.EnrichedCorrectionProposal{candidates[0], candidates[2]}
nextClient := &scriptedValidationClient{replies: []validationReply{
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}},
}}
nextValidator, _ := NewLLMBackedValidator("meaning_reversal_review", LLMValidatorTypeMeaningReversal)
nextValidator.SetTokenEstimator(perItemEstimator{})
@@ -101,7 +101,7 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
func TestValidationDomainInvalidAfterRepairIsAnError(t *testing.T) {
client := &scriptedValidationClient{replies: []validationReply{{
response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Reason: "ok"}}},
response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "99", Approved: true, Reason: "ok"}}},
completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 2, StructuralRepairs: 1}},
}}}
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
@@ -116,8 +116,8 @@ func TestValidationDomainInvalidAfterRepairIsAnError(t *testing.T) {
func TestValidationExtraDecisionsMergeActualRecoveries(t *testing.T) {
client := &scriptedValidationClient{replies: []validationReply{{
response: LLMValidationResponse{Validations: []LLMValidationDecision{
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
{CorrectionIndex: 99, Approved: true, Reason: "extra"},
{CorrectionIndex: "1", Approved: true, Reason: "ok"},
{CorrectionIndex: "99", Approved: true, Reason: "extra"},
}},
completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 3, TransientRetries: 1, StructuralRepairs: 1}},
}}}
@@ -137,9 +137,9 @@ func TestValidationExtraDecisionsMergeActualRecoveries(t *testing.T) {
func TestValidationFatalFailureKeepsEarlierFactsAndStops(t *testing.T) {
cause := errors.New("broken embedded schema")
client := &scriptedValidationClient{replies: []validationReply{
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}},
{err: &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "model setup failed", Cause: cause}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}},
}}
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
validator.SetTokenEstimator(perItemEstimator{})

View File

@@ -2,6 +2,7 @@ package validators
import (
"context"
"encoding/json"
"errors"
"fmt"
"sort"
@@ -9,6 +10,7 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/jsonnumber"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/stagename"
@@ -58,8 +60,8 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
if req.LLMClient == nil {
return Result{}, fmt.Errorf("LLM-backed validator %q requires a structured LLM client", v.name)
}
validationReq, immediate := BuildLLMValidationRequest(v.name, v.validatorType, req)
if len(validationReq.Items) == 0 {
items, immediate := buildValidationItems(req)
if len(items) == 0 {
all := append([]Decision(nil), immediate...)
sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex })
if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil {
@@ -73,7 +75,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
maxTokens = req.Settings.Validation.MaxPromptTokens
}
batches, oversized, err := prepareValidationBatches(validationReq.Items, maxTokens, v.estimator)
batches, oversized, err := prepareValidationBatches(items, maxTokens, v.estimator)
operations := make([]outcomes.Operation, 0)
llmDecisions := make([]Decision, 0)
partialResult := func() Result {
@@ -239,7 +241,7 @@ func errPayload(err error) any {
return map[string]any{"error": err.Error()}
}
func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorType, req Request) (LLMValidationRequest, []Decision) {
func buildValidationItems(req Request) ([]LLMValidationItem, []Decision) {
items := make([]LLMValidationItem, 0, len(req.CandidateProposal))
immediate := make([]Decision, 0)
req.Section = req.sectionLookup()
@@ -268,7 +270,6 @@ func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorT
neighbors, glossaryEntries := glossary.validationContext(segments, position, preview.CorrectedSegmentText)
items = append(items, LLMValidationItem{
CorrectionIndex: p.ProposalIndex,
SegmentID: p.TargetSegmentID,
Speaker: seg.Speaker,
OriginalText: p.OriginalText,
CorrectedText: p.CorrectedText,
@@ -280,31 +281,25 @@ func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorT
})
}
return LLMValidationRequest{
ValidatorName: validatorName,
ValidatorType: validatorType,
ModuleKey: req.ModuleKey,
ModuleInstance: req.ModuleInstance,
ReplacementPolicy: string(req.ReplacementPolicy),
Items: items,
}, immediate
return items, immediate
}
type validationReconciliation struct {
Decisions []Decision `json:"decisions"`
UnavailableIndices []int `json:"unavailable_indices,omitempty"`
IgnoredIndices []int `json:"ignored_indices,omitempty"`
Decisions []Decision `json:"decisions"`
UnavailableIndices []int `json:"unavailable_indices,omitempty"`
IgnoredIndices []json.Number `json:"ignored_indices,omitempty"`
}
func reconcileLLMResponse(items []LLMValidationItem, response LLMValidationResponse) validationReconciliation {
result := validationReconciliation{Decisions: make([]Decision, 0, len(items))}
seen := make(map[int][]LLMValidationDecision, len(items))
for _, d := range response.Validations {
if d.CorrectionIndex < 1 || d.CorrectionIndex > len(items) {
index, ok := jsonnumber.PositiveIndex(d.CorrectionIndex.String(), len(items))
if !ok {
result.IgnoredIndices = append(result.IgnoredIndices, d.CorrectionIndex)
continue
}
seen[d.CorrectionIndex] = append(seen[d.CorrectionIndex], d)
seen[index] = append(seen[index], d)
}
for i, item := range items {

View File

@@ -4,6 +4,8 @@ import (
"context"
"encoding/json"
"errors"
"slices"
"strconv"
"strings"
"testing"
@@ -74,7 +76,7 @@ func mk(index int, orig, corr string) proposals.EnrichedCorrectionProposal {
}
}
func TestBuildLLMValidationRequestPreservesLiteralDeletionAndSpacing(t *testing.T) {
func TestValidationPayloadPreservesLiteralDeletionAndSpacing(t *testing.T) {
for _, tc := range []struct {
name, segment, original, replacement, result string
}{
@@ -84,27 +86,25 @@ func TestBuildLLMValidationRequestPreservesLiteralDeletionAndSpacing(t *testing.
t.Run(tc.name, func(t *testing.T) {
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(7, tc.original, tc.replacement)})
req.WorkingTranscript.Segments[0].Text = tc.segment
payload, immediate := BuildLLMValidationRequest("editorial_review", LLMValidatorTypeEditorialReview, req)
if len(immediate) != 0 || len(payload.Items) != 1 {
payload, immediate := buildValidationItems(req)
if len(immediate) != 0 || len(payload) != 1 {
t.Fatalf("expected previewable request: %+v, %+v", payload, immediate)
}
raw, err := json.Marshal(payload)
variables, err := BuildValidationVariables(payload, "")
if err != nil {
t.Fatal(err)
}
var decoded struct {
ValidatorName string `json:"validator_name"`
Corrections []map[string]any `json:"corrections"`
}
raw := []byte(variables["PayloadJSON"])
var decoded []map[string]any
if err := json.Unmarshal(raw, &decoded); err != nil {
t.Fatal(err)
}
if decoded.ValidatorName != "editorial_review" || len(decoded.Corrections) != 1 {
t.Fatalf("wrong validator identity or item count: %s", raw)
if len(decoded) != 1 {
t.Fatalf("wrong item count: %s", raw)
}
item := decoded.Corrections[0]
item := decoded[0]
value, present := item["corrected_text"]
if !present || value != tc.replacement || item["original_text"] != tc.original || item["original_segment_text"] != tc.segment || item["corrected_segment_text"] != tc.result || item["correction_index"] != float64(7) {
if !present || value != tc.replacement || item["original_text"] != tc.original || item["original_segment_text"] != tc.segment || item["corrected_segment_text"] != tc.result || item["correction_index"] != float64(1) {
t.Fatalf("literal spans or full segment text lost: %s", raw)
}
})
@@ -148,8 +148,8 @@ func TestChunkLLMValidationItemsMultipleBatchesStableNoDropNoDup(t *testing.T) {
func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
{CorrectionIndex: 1, Approved: true, Reason: ""},
{CorrectionIndex: 2, Approved: false, Reason: "bad"},
{CorrectionIndex: "1", Approved: true, Reason: ""},
{CorrectionIndex: "2", Approved: false, Reason: "bad"},
}}}}
v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
if err != nil {
@@ -203,10 +203,10 @@ func TestLLMBackedValidatorProviderMalformedEnvelopePropagates(t *testing.T) {
func TestLLMBackedValidatorReconcilesDecisionIdentities(t *testing.T) {
approve := func(index int) LLMValidationDecision {
return LLMValidationDecision{CorrectionIndex: index, Approved: true, Reason: "accepted"}
return LLMValidationDecision{CorrectionIndex: json.Number(strconv.Itoa(index)), Approved: true, Reason: "accepted"}
}
reject := func(index int) LLMValidationDecision {
return LLMValidationDecision{CorrectionIndex: index, Reason: "editorial concern"}
return LLMValidationDecision{CorrectionIndex: json.Number(strconv.Itoa(index)), Reason: "editorial concern"}
}
type expectedDecision struct {
index int
@@ -218,13 +218,13 @@ func TestLLMBackedValidatorReconcilesDecisionIdentities(t *testing.T) {
entries []LLMValidationDecision
want []expectedDecision
unavailable []int
ignored []int
ignored []json.Number
}{
{"internal index is not a local reference", []LLMValidationDecision{approve(7), approve(2), reject(3)}, []expectedDecision{{7, false, ReasonMissingValidationDecision}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, []int{7}, []int{7}},
{"missing and extra", []LLMValidationDecision{approve(1), reject(3), approve(99)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonMissingValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, []int{99}},
{"internal index is not a local reference", []LLMValidationDecision{approve(7), approve(2), reject(3)}, []expectedDecision{{7, false, ReasonMissingValidationDecision}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, []int{7}, []json.Number{"7"}},
{"missing and extra", []LLMValidationDecision{approve(1), reject(3), approve(99)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonMissingValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, []json.Number{"99"}},
{"conflicting duplicate", []LLMValidationDecision{approve(1), approve(2), reject(2), reject(3)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonDuplicateValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
{"identical duplicate", []LLMValidationDecision{approve(1), approve(2), approve(2), reject(3)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonDuplicateValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
{"extra only", []LLMValidationDecision{approve(99), reject(3), approve(1), approve(99), approve(2)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, []int{99, 99}},
{"extra only", []LLMValidationDecision{approve(99), reject(3), approve(1), approve(99), approve(2)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, []json.Number{"99", "99"}},
{"empty", nil, []expectedDecision{{7, false, ReasonMissingValidationDecision}, {11, false, ReasonMissingValidationDecision}, {18, false, ReasonMissingValidationDecision}}, []int{7, 11, 18}, nil},
{"reordered", []LLMValidationDecision{reject(3), approve(2), approve(1)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, nil},
} {
@@ -264,7 +264,7 @@ func TestLLMBackedValidatorReconcilesDecisionIdentities(t *testing.T) {
t.Fatalf("supplied model decisions lost from diagnostic: %+v", decoded)
}
reconciliation := payload["reconciliation"].(validationReconciliation)
if !equalIndices(reconciliation.UnavailableIndices, tc.unavailable) || !equalIndices(reconciliation.IgnoredIndices, tc.ignored) || len(reconciliation.Decisions) != len(tc.want) {
if !equalIndices(reconciliation.UnavailableIndices, tc.unavailable) || !slices.Equal(reconciliation.IgnoredIndices, tc.ignored) || len(reconciliation.Decisions) != len(tc.want) {
t.Fatalf("diagnostic reconciliation=%+v", reconciliation)
}
})
@@ -285,7 +285,7 @@ func equalIndices(a, b []int) bool {
func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
{CorrectionIndex: "1", Approved: true, Reason: "ok"},
}}}}
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
huge := strings.Repeat("gestures ", 200)
@@ -331,7 +331,7 @@ func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *tes
func TestLLMBackedValidatorDiagnosticsIncludeSchemaMetadata(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
{CorrectionIndex: "1", Approved: true, Reason: "ok"},
}}}}
writer := &captureValidationDiagnosticsWriter{}
v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
@@ -370,7 +370,7 @@ func TestEveryValidationTypeSelectsEmbeddedPromptAndVariables(t *testing.T) {
}
for _, tc := range cases {
t.Run(string(tc.kind), func(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}}}
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}}}
validator, err := NewLLMBackedValidator(string(tc.kind), tc.kind)
if err != nil {
t.Fatal(err)
@@ -395,7 +395,7 @@ func TestEveryValidationTypeSelectsEmbeddedPromptAndVariables(t *testing.T) {
func TestValidationDiagnosticFailureDoesNotChangeDecision(t *testing.T) {
cause := errors.New("diagnostic storage unavailable")
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}}}
req.LLMClient = &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}}}
sink := &captureValidationDiagnosticsWriter{err: cause}
req.Interactions = sink
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)

View File

@@ -62,6 +62,25 @@ type Decision struct {
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
}
// EffectiveDisposition resolves an omitted disposition from the declared
// approval and detects inconsistent explicit decisions.
func (d Decision) EffectiveDisposition() (DecisionDisposition, error) {
switch d.Disposition {
case "":
if d.Approved {
return DispositionApproved, nil
}
return DispositionRejected, nil
case DispositionApproved, DispositionRejected, DispositionUnavailable:
if d.Approved == (d.Disposition == DispositionApproved) {
return d.Disposition, nil
}
return "", fmt.Errorf("contradictory approval and disposition %q for proposal %d", d.Disposition, d.ProposalIndex)
default:
return "", fmt.Errorf("invalid decision disposition %q for proposal %d", d.Disposition, d.ProposalIndex)
}
}
// DecisionDisposition separates normal editorial rejection from a candidate
// excluded because its model operation could not provide a trustworthy decision.
type DecisionDisposition string
@@ -80,7 +99,10 @@ type Result struct {
Operations []outcomes.Operation `json:"operations,omitempty"`
}
// Validator is the deterministic runtime validator interface.
// Validator evaluates one section's candidates. An instance may be called
// concurrently across sections and may keep synchronized state. Request data
// is borrowed for the call; inputs must be copied before modification or
// retention. Request.Preview may populate the caller's pass-local cache.
type Validator interface {
Name() string
Validate(ctx context.Context, req Request) (Result, error)

View File

@@ -54,3 +54,21 @@ func (r Request) Preview(candidate proposals.EnrichedCorrectionProposal) proposa
}
return preview
}
// AcceptsIndependentCandidate decides which evidence can be submitted when
// repair needs findings from each applicable validator.
func (r Request) AcceptsIndependentCandidate(validatorName string, modelDriven bool, candidate proposals.EnrichedCorrectionProposal) bool {
hasSpan := candidate.OriginalText != ""
switch validatorName {
case "proposal_shape":
return true
case "no_effect":
return hasSpan
case "original_text_presence":
return hasSpan && candidate.TargetSegmentID > 0
case "protected_terms", "non_empty_resulting_segment":
return r.Preview(candidate).HasResult()
default:
return !modelDriven || r.Preview(candidate).Applicable
}
}

View File

@@ -2,6 +2,7 @@ package glossary
import (
"context"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
@@ -11,7 +12,7 @@ import (
)
type Module struct {
validators []contracts.Validator
validators []frameworkvalidators.Validator
}
func New() (*Module, error) {
@@ -33,8 +34,8 @@ func (m *Module) ReplacementPolicy() proposals.ReplacementPolicy {
return proposals.ReplacementPolicyReplaceAll
}
func (m *Module) Validators() []contracts.Validator {
return append([]contracts.Validator(nil), m.validators...)
func (m *Module) Validators() []frameworkvalidators.Validator {
return append([]frameworkvalidators.Validator(nil), m.validators...)
}
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {

View File

@@ -2,6 +2,7 @@ package grammar
import (
"context"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
@@ -11,7 +12,7 @@ import (
)
type Module struct {
validators []contracts.Validator
validators []frameworkvalidators.Validator
}
func New() (*Module, error) {
@@ -33,8 +34,8 @@ func (m *Module) ReplacementPolicy() proposals.ReplacementPolicy {
return proposals.ReplacementPolicyRequireUnique
}
func (m *Module) Validators() []contracts.Validator {
return append([]contracts.Validator(nil), m.validators...)
func (m *Module) Validators() []frameworkvalidators.Validator {
return append([]frameworkvalidators.Validator(nil), m.validators...)
}
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {

View File

@@ -2,6 +2,7 @@ package homophones
import (
"context"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
@@ -11,7 +12,7 @@ import (
)
type Module struct {
validators []contracts.Validator
validators []frameworkvalidators.Validator
}
func New() (*Module, error) {
@@ -33,8 +34,8 @@ func (m *Module) ReplacementPolicy() proposals.ReplacementPolicy {
return proposals.ReplacementPolicyRequireUnique
}
func (m *Module) Validators() []contracts.Validator {
return append([]contracts.Validator(nil), m.validators...)
func (m *Module) Validators() []frameworkvalidators.Validator {
return append([]frameworkvalidators.Validator(nil), m.validators...)
}
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {

View File

@@ -154,7 +154,7 @@ func TestHomophonesProtectedGlossaryTermBehaviorRejectsUnsafeCorrection(t *testi
if err != nil {
t.Fatalf("New error: %v", err)
}
var protectedValidator contracts.Validator
var protectedValidator validators.Validator
for _, v := range module.Validators() {
if v.Name() == "protected_terms" {
protectedValidator = v

View File

@@ -2,6 +2,7 @@ package spoken_word
import (
"context"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
@@ -11,7 +12,7 @@ import (
)
type Module struct {
validators []contracts.Validator
validators []frameworkvalidators.Validator
}
func New() (*Module, error) {
@@ -33,8 +34,8 @@ func (m *Module) ReplacementPolicy() proposals.ReplacementPolicy {
return proposals.ReplacementPolicyRequireUnique
}
func (m *Module) Validators() []contracts.Validator {
return append([]contracts.Validator(nil), m.validators...)
func (m *Module) Validators() []frameworkvalidators.Validator {
return append([]frameworkvalidators.Validator(nil), m.validators...)
}
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {

View File

@@ -154,7 +154,7 @@ func TestSpokenWordProtectedGlossaryTermBehaviorRejectsUnsafeCorrection(t *testi
if err != nil {
t.Fatalf("New error: %v", err)
}
var protectedValidator contracts.Validator
var protectedValidator validators.Validator
for _, v := range module.Validators() {
if v.Name() == "protected_terms" {
protectedValidator = v

View File

@@ -2,10 +2,10 @@ package validators
import (
"fmt"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
"strings"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
)
var builtInChains = map[string][]string{
@@ -58,7 +58,7 @@ func BuiltInChainKeys(moduleKey string) ([]string, error) {
return out, nil
}
func ResolveBuiltInChain(moduleKey string, registry *Registry) ([]contracts.Validator, error) {
func ResolveBuiltInChain(moduleKey string, registry *Registry) ([]frameworkvalidators.Validator, error) {
moduleKey = strings.TrimSpace(moduleKey)
keys, err := BuiltInChainKeys(moduleKey)
if err != nil {
@@ -68,7 +68,7 @@ func ResolveBuiltInChain(moduleKey string, registry *Registry) ([]contracts.Vali
registry = NewBuiltInRegistry()
}
out := make([]contracts.Validator, 0, len(keys))
out := make([]frameworkvalidators.Validator, 0, len(keys))
for _, key := range keys {
v, buildErr := registry.MustBuild(key)
if buildErr != nil {

View File

@@ -1,14 +1,13 @@
package editorial_review
import (
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
)
const Key = "editorial_review"
func New() (contracts.Validator, error) {
func New() (frameworkvalidators.Validator, error) {
v, err := frameworkvalidators.NewLLMBackedValidator(Key, frameworkvalidators.LLMValidatorTypeEditorialReview)
if err != nil {
return nil, err

View File

@@ -1,14 +1,13 @@
package meaning_reversal_review
import (
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
)
const Key = "meaning_reversal_review"
func New() (contracts.Validator, error) {
func New() (frameworkvalidators.Validator, error) {
v, err := frameworkvalidators.NewLLMBackedValidator(Key, frameworkvalidators.LLMValidatorTypeMeaningReversal)
if err != nil {
return nil, err

View File

@@ -3,7 +3,7 @@ package metadata
import (
"strings"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
)
type ExecutionClass string
@@ -37,11 +37,11 @@ var executionClassByKey = map[string]ExecutionClass{
}
type ClassifiedValidator interface {
contracts.Validator
frameworkvalidators.Validator
ExecutionClass() ExecutionClass
}
func ClassOf(v contracts.Validator) ExecutionClass {
func ClassOf(v frameworkvalidators.Validator) ExecutionClass {
if v == nil {
return ExecutionClassDeterministic
}
@@ -69,7 +69,7 @@ func ClassForKey(key string) ExecutionClass {
return class
}
func Wrap(v contracts.Validator, class ExecutionClass) contracts.Validator {
func Wrap(v frameworkvalidators.Validator, class ExecutionClass) frameworkvalidators.Validator {
if v == nil {
return nil
}
@@ -83,7 +83,7 @@ func Wrap(v contracts.Validator, class ExecutionClass) contracts.Validator {
}
type wrappedValidator struct {
contracts.Validator
frameworkvalidators.Validator
class ExecutionClass
}

View File

@@ -4,7 +4,6 @@ import (
"context"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
)
@@ -12,7 +11,7 @@ type unclassifiedValidator struct{}
func (u unclassifiedValidator) Name() string { return "unclassified" }
func (u unclassifiedValidator) Validate(_ context.Context, _ contracts.ValidationRequest) (frameworkvalidators.Result, error) {
func (u unclassifiedValidator) Validate(_ context.Context, _ frameworkvalidators.Request) (frameworkvalidators.Result, error) {
return frameworkvalidators.Result{ValidatorName: u.Name(), Decisions: nil}, nil
}
@@ -22,7 +21,7 @@ type namedUnclassifiedValidator struct {
func (n namedUnclassifiedValidator) Name() string { return n.name }
func (n namedUnclassifiedValidator) Validate(_ context.Context, _ contracts.ValidationRequest) (frameworkvalidators.Result, error) {
func (n namedUnclassifiedValidator) Validate(_ context.Context, _ frameworkvalidators.Request) (frameworkvalidators.Result, error) {
return frameworkvalidators.Result{ValidatorName: n.Name(), Decisions: nil}, nil
}

View File

@@ -3,7 +3,7 @@ package validators_test
import (
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
"gitea.maximumdirect.net/eric/audita/v3/internal/modules/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/modules/grammar"
"gitea.maximumdirect.net/eric/audita/v3/internal/modules/homophones"
@@ -20,30 +20,30 @@ func TestProductionModulesUseRegisteredValidatorKeys(t *testing.T) {
cases := []struct {
name string
build func() ([]contracts.Validator, error)
build func() ([]frameworkvalidators.Validator, error)
}{
{name: "glossary", build: func() ([]contracts.Validator, error) {
{name: "glossary", build: func() ([]frameworkvalidators.Validator, error) {
m, err := glossary.New()
if err != nil {
return nil, err
}
return m.Validators(), nil
}},
{name: "homophones", build: func() ([]contracts.Validator, error) {
{name: "homophones", build: func() ([]frameworkvalidators.Validator, error) {
m, err := homophones.New()
if err != nil {
return nil, err
}
return m.Validators(), nil
}},
{name: "spoken_word", build: func() ([]contracts.Validator, error) {
{name: "spoken_word", build: func() ([]frameworkvalidators.Validator, error) {
m, err := spoken_word.New()
if err != nil {
return nil, err
}
return m.Validators(), nil
}},
{name: "grammar", build: func() ([]contracts.Validator, error) {
{name: "grammar", build: func() ([]frameworkvalidators.Validator, error) {
m, err := grammar.New()
if err != nil {
return nil, err

View File

@@ -1,13 +1,12 @@
package no_effect
import (
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
)
const Key = "no_effect"
func New() (contracts.Validator, error) {
func New() (frameworkvalidators.Validator, error) {
return validatormetadata.Wrap(frameworkvalidators.NoEffectValidator{}, validatormetadata.ExecutionClassDeterministic), nil
}

View File

@@ -1,13 +1,12 @@
package non_empty_resulting_segment
import (
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
)
const Key = "non_empty_resulting_segment"
func New() (contracts.Validator, error) {
func New() (frameworkvalidators.Validator, error) {
return validatormetadata.Wrap(frameworkvalidators.NonEmptyResultingSegmentValidator{}, validatormetadata.ExecutionClassDeterministic), nil
}

View File

@@ -1,13 +1,12 @@
package original_text_presence
import (
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
)
const Key = "original_text_presence"
func New() (contracts.Validator, error) {
func New() (frameworkvalidators.Validator, error) {
return validatormetadata.Wrap(frameworkvalidators.OriginalTextPresenceValidator{}, validatormetadata.ExecutionClassDeterministic), nil
}

View File

@@ -1,13 +1,12 @@
package proposal_shape
import (
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
)
const Key = "proposal_shape"
func New() (contracts.Validator, error) {
func New() (frameworkvalidators.Validator, error) {
return validatormetadata.Wrap(frameworkvalidators.ProposalShapeValidator{}, validatormetadata.ExecutionClassDeterministic), nil
}

View File

@@ -3,14 +3,13 @@ package protected_terms
import (
"context"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
)
const Key = "protected_terms"
func New() (contracts.Validator, error) {
func New() (frameworkvalidators.Validator, error) {
return validatormetadata.Wrap(validator{}, validatormetadata.ExecutionClassDeterministic), nil
}
@@ -18,6 +17,6 @@ type validator struct{}
func (v validator) Name() string { return Key }
func (v validator) Validate(_ context.Context, req contracts.ValidationRequest) (frameworkvalidators.Result, error) {
func (v validator) Validate(_ context.Context, req frameworkvalidators.Request) (frameworkvalidators.Result, error) {
return frameworkvalidators.ValidateProtectedTerms(req)
}

View File

@@ -6,7 +6,6 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
@@ -50,7 +49,7 @@ func TestNewUsesGeneralBehaviorForGlossaryModule(t *testing.T) {
t.Fatalf("New: %v", err)
}
req := contracts.ValidationRequest{
req := frameworkvalidators.Request{
ModuleKey: "glossary",
Glossary: tinyGlossary(),
CandidateProposal: []proposals.EnrichedCorrectionProposal{
@@ -78,7 +77,7 @@ func TestNewRejectsProtectedTermChangeOutsideGlossaryStage(t *testing.T) {
t.Fatalf("New: %v", err)
}
req := contracts.ValidationRequest{
req := frameworkvalidators.Request{
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Jesters"}}},
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
ModuleKey: "homophones",
@@ -115,7 +114,7 @@ func TestNewGlossaryStageRejectsProtectedTermChange(t *testing.T) {
t.Fatalf("expected deterministic class, got %q", got)
}
req := contracts.ValidationRequest{
req := frameworkvalidators.Request{
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Jesters"}}},
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
ModuleKey: "glossary",
@@ -146,7 +145,7 @@ func TestNewGlossaryStageAllowsProtectedTermInsertion(t *testing.T) {
t.Fatalf("New: %v", err)
}
req := contracts.ValidationRequest{
req := frameworkvalidators.Request{
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "gestures"}}},
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
ModuleKey: "glossary",

View File

@@ -2,9 +2,9 @@ package validators
import (
"fmt"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
"strings"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/validators/editorial_review"
"gitea.maximumdirect.net/eric/audita/v3/internal/validators/meaning_reversal_review"
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
@@ -30,7 +30,7 @@ const (
type BuiltInValidatorDefinition struct {
Key string
Build func() (contracts.Validator, error)
Build func() (frameworkvalidators.Validator, error)
}
type Registry struct {
@@ -64,7 +64,7 @@ func (r *Registry) Lookup(key string) (BuiltInValidatorDefinition, bool) {
return def, ok
}
func (r *Registry) MustBuild(key string) (contracts.Validator, error) {
func (r *Registry) MustBuild(key string) (frameworkvalidators.Validator, error) {
if r == nil {
return nil, fmt.Errorf("validator registry is nil")
}

View File

@@ -2,12 +2,12 @@ package validators
import (
"context"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/validators/editorial_review"
"gitea.maximumdirect.net/eric/audita/v3/internal/validators/meaning_reversal_review"
@@ -50,7 +50,7 @@ func TestBuiltInValidatorPackagesConstruct(t *testing.T) {
type validatorCtor struct {
name string
key string
build func() (contracts.Validator, error)
build func() (frameworkvalidators.Validator, error)
wantClass validatormetadata.ExecutionClass
}
cases := []validatorCtor{
@@ -117,7 +117,7 @@ func TestRegistryProtectedTermsUsesNonGlossaryStageBehavior(t *testing.T) {
t.Fatalf("must build protected_terms: %v", err)
}
req := contracts.ValidationRequest{
req := frameworkvalidators.Request{
ModuleKey: "glossary",
Glossary: []glossary.Entry{{Collection: "party", ID: "jesters", Name: "Jesters", Category: "faction", Description: "Protected"}},
CandidateProposal: []proposals.EnrichedCorrectionProposal{
@@ -152,7 +152,7 @@ func TestResolveBuiltInChainUsesGlossaryStageProtectedTermsBehavior(t *testing.T
t.Fatalf("resolve glossary chain: %v", err)
}
var v contracts.Validator
var v frameworkvalidators.Validator
for _, candidate := range chain {
if candidate.Name() == KeyProtectedTerms {
v = candidate
@@ -163,7 +163,7 @@ func TestResolveBuiltInChainUsesGlossaryStageProtectedTermsBehavior(t *testing.T
t.Fatalf("expected protected_terms validator in glossary chain")
}
req := contracts.ValidationRequest{
req := frameworkvalidators.Request{
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Jesters"}}},
ReplacementPolicy: proposals.ReplacementPolicyReplaceAll,
ModuleKey: "glossary",

View File

@@ -1,14 +1,13 @@
package spoken_form_plausibility
import (
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
)
const Key = "spoken_form_plausibility"
func New() (contracts.Validator, error) {
func New() (frameworkvalidators.Validator, error) {
v, err := frameworkvalidators.NewLLMBackedValidator(Key, frameworkvalidators.LLMValidatorTypeSpokenFormPlausibility)
if err != nil {
return nil, err