Validate internal decisions and reconcile exact model references

This commit is contained in:
2026-11-16 14:11:02 +00:00
parent 475557e4de
commit 0c144fd5a5
46 changed files with 544 additions and 376 deletions

View File

@@ -0,0 +1,82 @@
// Package jsonnumber converts exact integral JSON tokens without float rounding.
package jsonnumber
import (
"strconv"
"strings"
)
// Int accepts lexical integers and equivalent decimal and exponent spellings
// in the native integer range. value must be a valid JSON number token.
func Int(value string) (int, bool) {
if integer, err := strconv.Atoi(value); err == nil {
return integer, true
}
digits, ok := integralDecimal(value)
if !ok {
return 0, false
}
integer, err := strconv.Atoi(digits)
return integer, err == nil
}
// PositiveIndex accepts only references in [1, maximum]. Comparison with the
// requested range precedes conversion to the local integer index.
func PositiveIndex(value string, maximum int) (int, bool) {
if strings.ContainsAny(value, ".eE") {
var ok bool
if value, ok = integralDecimal(value); !ok {
return 0, false
}
}
if maximum < 1 || value == "" || value[0] < '1' || value[0] > '9' {
return 0, false
}
limit := strconv.Itoa(maximum)
if len(value) > len(limit) || (len(value) == len(limit) && value > limit) {
return 0, false
}
index, err := strconv.Atoi(value)
return index, err == nil
}
func integralDecimal(value string) (string, bool) {
if value == "" || (value[0] != '-' && (value[0] < '0' || value[0] > '9')) || !strings.ContainsAny(value, ".eE") {
return "", false
}
digits, negative := strings.CutPrefix(value, "-")
var exponentText string
if i := strings.IndexAny(digits, "eE"); i >= 0 {
exponentText, digits = digits[i+1:], digits[:i]
}
whole, fraction, _ := strings.Cut(digits, ".")
digits = strings.TrimLeft(whole+fraction, "0")
if digits == "" {
return "0", true
}
var exponent int
if exponentText != "" {
var err error
exponent, err = strconv.Atoi(exponentText)
if err != nil || exponent < -len(digits) || exponent > len(fraction)+19 {
return "", false
}
}
shift := exponent - len(fraction)
if shift < 0 {
// A negative shift is integral only if every discarded digit is zero.
if -shift > len(digits)-len(strings.TrimRight(digits, "0")) {
return "", false
}
digits = digits[:len(digits)+shift]
} else {
if len(digits)+shift > 19 {
return "", false
}
digits += strings.Repeat("0", shift)
}
if negative {
digits = "-" + digits
}
return digits, true
}

View File

@@ -6,8 +6,8 @@ import (
"fmt"
"math"
"slices"
"strconv"
"strings"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/jsonnumber"
)
type SourceSegment struct {
@@ -114,60 +114,13 @@ func extractSegments(raw []byte) (json.RawMessage, error) {
type sourceID int
func (id *sourceID) UnmarshalJSON(raw []byte) error {
spelling := string(raw)
if value, err := strconv.Atoi(spelling); err == nil {
*id = sourceID(value)
return nil
}
if value, ok := integralDecimal(spelling); ok {
if value, ok := jsonnumber.Int(string(raw)); ok {
*id = sourceID(value)
return nil
}
return json.Unmarshal(raw, (*int)(id))
}
func integralDecimal(value string) (int, bool) {
if value == "" || (value[0] != '-' && (value[0] < '0' || value[0] > '9')) || !strings.ContainsAny(value, ".eE") {
return 0, false
}
digits, negative := strings.CutPrefix(value, "-")
var exponentText string
if i := strings.IndexAny(digits, "eE"); i >= 0 {
exponentText, digits = digits[i+1:], digits[:i]
}
whole, fraction, _ := strings.Cut(digits, ".")
digits = strings.TrimLeft(whole+fraction, "0")
if digits == "" {
return 0, true
}
var exponent int
if exponentText != "" {
var err error
exponent, err = strconv.Atoi(exponentText)
if err != nil || exponent < -len(digits) || exponent > len(fraction)+19 {
return 0, false
}
}
shift := exponent - len(fraction)
if shift < 0 {
// A negative shift is integral only if every discarded digit is zero.
if -shift > len(digits)-len(strings.TrimRight(digits, "0")) {
return 0, false
}
digits = digits[:len(digits)+shift]
} else {
if len(digits)+shift > 19 {
return 0, false
}
digits += strings.Repeat("0", shift)
}
if negative {
digits = "-" + digits
}
integer, err := strconv.Atoi(digits)
return integer, err == nil
}
func validateSourceSegments(segments []SourceSegment) error {
for i, s := range segments {
segLabel := segmentLabel(i, s.ID)

View File

@@ -9,6 +9,7 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
)
// testChunkProposalHarness is a test-only helper that composes existing
@@ -77,7 +78,7 @@ func (m deterministicFakeModule) ReplacementPolicy() proposals.ReplacementPolicy
return m.replacementPolicy
}
func (m deterministicFakeModule) Validators() []Validator { return nil }
func (m deterministicFakeModule) Validators() []validators.Validator { return nil }
func (m deterministicFakeModule) Propose(ctx context.Context, req ProposalRequest) (ProposalResult, error) {
_ = ctx

View File

@@ -22,16 +22,10 @@ type StructuredLLMClient = structuredoutput.Client
type TranscriptModule interface {
Key() string
ReplacementPolicy() proposals.ReplacementPolicy
Validators() []Validator
Validators() []validators.Validator
Propose(ctx context.Context, req ProposalRequest) (ProposalResult, error)
}
// Validator evaluates candidate proposals and returns one decision per proposal index.
type Validator interface {
Name() string
Validate(ctx context.Context, req ValidationRequest) (validators.Result, error)
}
type StructuredCompletionRequest = structuredoutput.Request
type StructuredCompletionResponse = structuredoutput.Response
@@ -98,9 +92,6 @@ type ProposalResult struct {
Operations []outcomes.Operation `json:"operations,omitempty"`
}
// ValidationRequest is the input to validator execution.
type ValidationRequest = validators.Request
// ResolveModuleRunSpecs derives isolated consumer views from resolved settings.
func ResolveModuleRunSpecs(cfg *config.Config) []ModuleRunSpec {
if cfg == nil {

View File

@@ -30,7 +30,7 @@ type fakeValidator struct{}
func (f *fakeValidator) Name() string { return "fake-validator" }
func (f *fakeValidator) Validate(ctx context.Context, req ValidationRequest) (validators.Result, error) {
func (f *fakeValidator) Validate(ctx context.Context, req validators.Request) (validators.Result, error) {
_ = ctx
decisions := make([]validators.Decision, len(req.CandidateProposal))
for i, proposal := range req.CandidateProposal {
@@ -47,8 +47,8 @@ func (f *fakeModule) ReplacementPolicy() proposals.ReplacementPolicy {
return proposals.ReplacementPolicyRequireUnique
}
func (f *fakeModule) Validators() []Validator {
return []Validator{&fakeValidator{}}
func (f *fakeModule) Validators() []validators.Validator {
return []validators.Validator{&fakeValidator{}}
}
func (f *fakeModule) Propose(ctx context.Context, req ProposalRequest) (ProposalResult, error) {
@@ -63,7 +63,7 @@ func (f *fakeModule) Propose(ctx context.Context, req ProposalRequest) (Proposal
func TestInterfaceContractsCompileWithFakes(t *testing.T) {
var _ StructuredLLMClient = (*fakeLLMClient)(nil)
var _ Validator = (*fakeValidator)(nil)
var _ validators.Validator = (*fakeValidator)(nil)
var _ TranscriptModule = (*fakeModule)(nil)
module := &fakeModule{}

View File

@@ -3,6 +3,7 @@ package modules
import (
"context"
"errors"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
"strings"
"testing"
@@ -20,7 +21,7 @@ func (m noopModule) Key() string { return m.key }
func (m noopModule) ReplacementPolicy() proposals.ReplacementPolicy {
return proposals.ReplacementPolicyRequireUnique
}
func (m noopModule) Validators() []contracts.Validator { return nil }
func (m noopModule) Validators() []frameworkvalidators.Validator { return nil }
func (m noopModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
_ = ctx
_ = req

View File

@@ -2,7 +2,9 @@ package processreport
import (
"context"
"encoding/json"
"errors"
"strconv"
"strings"
"testing"
@@ -16,13 +18,15 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
)
type partialModule struct{ validator contracts.Validator }
type partialModule struct{ validator validators.Validator }
func (m partialModule) Key() string { return "grammar" }
func (m partialModule) ReplacementPolicy() proposals.ReplacementPolicy {
return proposals.ReplacementPolicyRequireUnique
}
func (m partialModule) Validators() []contracts.Validator { return []contracts.Validator{m.validator} }
func (m partialModule) Validators() []validators.Validator {
return []validators.Validator{m.validator}
}
func (m partialModule) ModuleForSpec(contracts.ModuleRunSpec) (contracts.TranscriptModule, error) {
return m, nil
}
@@ -55,7 +59,7 @@ func (c *interruptedValidationClient) CompleteStructured(_ context.Context, req
*target.(*validators.LLMValidationResponse) = validators.LLMValidationResponse{}
return structuredoutput.Response{}, nil
}
*target.(*validators.LLMValidationResponse) = validators.LLMValidationResponse{Validations: []validators.LLMValidationDecision{{CorrectionIndex: index, Approved: index == 1, Reason: "completed assessment"}}}
*target.(*validators.LLMValidationResponse) = validators.LLMValidationResponse{Validations: []validators.LLMValidationDecision{{CorrectionIndex: json.Number(strconv.Itoa(index)), Approved: index == 1, Reason: "completed assessment"}}}
return structuredoutput.Response{}, nil
}

View File

@@ -104,7 +104,7 @@ func TestValidatorFailureCancelsActiveAndPendingProviderWork(t *testing.T) {
}, &out)
return validators.Result{}, err
}}
module := fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{validator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
module := fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{validator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "hello", CorrectedText: "Hello"}}, nil
}}
sections := []chunking.Section{}

View File

@@ -1,48 +1,19 @@
package runner
import (
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
)
// feedbackCandidates submits only candidates for which the validator has usable evidence.
func feedbackCandidates(input validateSectionCandidatesInput, validator contracts.Validator, noEffect map[int]bool, evidence contracts.ValidationRequest) []proposals.EnrichedCorrectionProposal {
func feedbackCandidates(input validateSectionCandidatesInput, validatorName string, modelDriven bool, noEffect map[int]bool, evidence validators.Request) []proposals.EnrichedCorrectionProposal {
var result []proposals.EnrichedCorrectionProposal
for _, candidate := range input.SectionEnriched {
if noEffect[candidate.ProposalIndex] {
continue
}
usableSpan := candidate.OriginalText != ""
target := candidate.TargetSegmentID > 0
var applicable bool
switch validator.Name() {
case "proposal_shape":
applicable = true
case "no_effect":
applicable = usableSpan
case "original_text_presence":
applicable = usableSpan && target
case "protected_terms":
applicable = usableSpan
if input.Spec.ModuleKey == modulecatalog.KeyGlossary || input.Spec.ModuleKey == modulecatalog.KeySpokenWord {
preview := evidence.Preview(candidate)
applicable = preview.Applicable || preview.SkipReason == proposals.SkipReasonEmptyResultingText
}
case "non_empty_resulting_segment":
preview := evidence.Preview(candidate)
applicable = preview.Applicable || preview.SkipReason == proposals.SkipReasonEmptyResultingText
default:
if validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked {
applicable = evidence.Preview(candidate).Applicable
} else {
applicable = true
}
}
if applicable {
if evidence.AcceptsIndependentCandidate(validatorName, modelDriven, candidate) {
result = append(result, candidate)
}
}

View File

@@ -25,7 +25,7 @@ func TestFeedbackCollectsIndependentFindingsAndUsableSemanticInput(t *testing.T)
return proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: target, OriginalText: original, CorrectedText: replacement}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index}}
}
var seen []int
semantic := validatormetadata.Wrap(fakeValidator{name: "semantic", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
semantic := validatormetadata.Wrap(fakeValidator{name: "semantic", validateF: func(req validators.Request) (validators.Result, error) {
var decisions []validators.Decision
for _, p := range req.CandidateProposal {
seen = append(seen, p.ProposalIndex)
@@ -33,7 +33,7 @@ func TestFeedbackCollectsIndependentFindingsAndUsableSemanticInput(t *testing.T)
}
return validators.Result{Decisions: decisions}, nil
}}, validatormetadata.ExecutionClassLLMBacked)
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh teh cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, semantic}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate(0, 1, "cat", "dog"), candidate(1, 0, "teh", "the"), candidate(2, 1, "teh", "the"), candidate(3, 1, "cat", "cat")}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh teh cat"}}}, Validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, semantic}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate(0, 1, "cat", "dog"), candidate(1, 0, "teh", "the"), candidate(2, 1, "teh", "the"), candidate(3, 1, "cat", "cat")}}
result, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
@@ -58,14 +58,14 @@ func TestFeedbackKeepsUnavailableAndZeroBudgetShortlists(t *testing.T) {
for _, budget := range []int{0, 1} {
t.Run(fmt.Sprint(budget), func(t *testing.T) {
var later int
first := fakeValidator{name: "first", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
first := fakeValidator{name: "first", validateF: func(req validators.Request) (validators.Result, error) {
var decisions []validators.Decision
for _, p := range req.CandidateProposal {
decisions = append(decisions, validators.Decision{ProposalIndex: p.ProposalIndex, Disposition: validators.DispositionUnavailable, ReasonCode: "unavailable"})
}
return validators.Result{Decisions: decisions}, nil
}}
second := fakeValidator{name: "second", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
second := fakeValidator{name: "second", validateF: func(req validators.Request) (validators.Result, error) {
later = len(req.CandidateProposal)
var decisions []validators.Decision
for _, p := range req.CandidateProposal {
@@ -73,7 +73,7 @@ func TestFeedbackKeepsUnavailableAndZeroBudgetShortlists(t *testing.T) {
}
return validators.Result{Decisions: decisions}, nil
}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{first, second}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{first, second}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}}
result, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
@@ -95,8 +95,8 @@ func TestRepairFindingsUseSafeUnavailableEvidence(t *testing.T) {
func TestRevalidationArtifactsUseProposalAndAttemptCoordinates(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}},
}}
validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
if err != nil {
@@ -108,7 +108,7 @@ func TestRevalidationArtifactsUseProposalAndAttemptCoordinates(t *testing.T) {
var paths []string
for _, index := range []int{2, 3} {
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar_1", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validator}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, ValidationLLMClient: client, Interactions: sink, SectionIndex: &section, RepairProposalIndex: &index, RepairAttempt: &attempt}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar_1", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{validator}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, ValidationLLMClient: client, Interactions: sink, SectionIndex: &section, RepairProposalIndex: &index, RepairAttempt: &attempt}
result, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
@@ -149,11 +149,11 @@ func TestRevalidationArtifactsUseProposalAndAttemptCoordinates(t *testing.T) {
func TestFeedbackReportsProtectionAndBlankResult(t *testing.T) {
identity := glossary.Identity{Collection: "party", ID: "jesters"}
protected := fakeValidator{name: "protected_terms", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
protected := fakeValidator{name: "protected_terms", validateF: func(req validators.Request) (validators.Result, error) {
return validators.ValidateProtectedTerms(req)
}}
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "Jesters", CorrectedText: ""}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "spoken_word", ProtectedTerms: []glossary.Term{{Spelling: "Jesters", Identities: []glossary.Identity{identity}}}, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Jesters"}}}, Validators: []contracts.Validator{protected, validators.NonEmptyResultingSegmentValidator{}}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "spoken_word", ProtectedTerms: []glossary.Term{{Spelling: "Jesters", Identities: []glossary.Identity{identity}}}, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Jesters"}}}, Validators: []validators.Validator{protected, validators.NonEmptyResultingSegmentValidator{}}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}}
result, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)

View File

@@ -2,6 +2,7 @@ package runner
import (
"context"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
@@ -27,7 +28,7 @@ func (m orderedModule) Key() string { return m.spec.ModuleKey }
func (m orderedModule) ReplacementPolicy() proposals.ReplacementPolicy {
return proposals.ReplacementPolicyRequireUnique
}
func (m orderedModule) Validators() []contracts.Validator { return nil }
func (m orderedModule) Validators() []frameworkvalidators.Validator { return nil }
func (m orderedModule) Propose(_ context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
m.seen <- req
original, corrected := "party", "Party"

View File

@@ -24,6 +24,7 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
)
type repairClient struct {
@@ -52,7 +53,7 @@ func TestRepairBranchesRetargetAndPreserveOtherApprovals(t *testing.T) {
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 4}},
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 0, OriginalText: "x", CorrectedText: "y"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 5}},
}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}, {ID: 2, Text: "uh hello"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.NonEmptyResultingSegmentValidator{}}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}, {ID: 2, Text: "uh hello"}}}, Validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.NonEmptyResultingSegmentValidator{}}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{}}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
@@ -92,7 +93,7 @@ func TestRepairRequestCarriesLiteralPreview(t *testing.T) {
Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}},
Policy: tc.policy,
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 81, Text: tc.segment}}},
Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.NonEmptyResultingSegmentValidator{}, rejectUntilRepair()},
Validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.NonEmptyResultingSegmentValidator{}, rejectUntilRepair()},
SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate},
Exchange: &structuredoutput.Exchange{},
}
@@ -131,7 +132,7 @@ func TestRepairRequestRefreshesPreviewAfterRetarget(t *testing.T) {
`{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`,
}}
var validatedIDs []int
reject := fakeValidator{name: "editorial_review", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
reject := fakeValidator{name: "editorial_review", validateF: func(req validators.Request) (validators.Result, error) {
validatedIDs = append(validatedIDs, req.CandidateProposal[0].TargetSegmentID)
return validators.Result{Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Disposition: validators.DispositionRejected, ReasonCode: "llm_rejected", Message: "The context does not support this edit."}}}, nil
}}
@@ -139,9 +140,10 @@ func TestRepairRequestRefreshesPreviewAfterRetarget(t *testing.T) {
Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 2}}},
Policy: proposals.ReplacementPolicyRequireUnique,
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 81, Text: "cat"}, {ID: 104, Text: "uh hello"}}},
Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, reject},
Validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validatormetadata.Wrap(reject, validatormetadata.ExecutionClassDeterministic)},
SectionEnriched: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 81, OriginalText: "cat", CorrectedText: "dog"}}},
Exchange: &structuredoutput.Exchange{},
Interactions: llm.NewDiagnosticsWriter(t.TempDir()),
}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
@@ -154,6 +156,15 @@ func TestRepairRequestRefreshesPreviewAfterRetarget(t *testing.T) {
if len(validatedIDs) != 2 || validatedIDs[0] != 81 || validatedIDs[1] != 104 {
t.Fatalf("validated segment IDs: %v", validatedIDs)
}
for _, decisions := range [][]ValidatorDecisionRecord{initial.decisions, branches[0].Attempts[0].Decisions} {
if len(decisions) == 0 {
t.Fatal("missing decision evidence")
}
record := decisions[len(decisions)-1]
if record.ValidatorName != "editorial_review" || record.ExecutionClass != validatormetadata.ExecutionClassDeterministic {
t.Fatalf("lost execution class override: %+v", record)
}
}
for i, want := range []struct{ before, after string }{{"cat", "dog"}, {"uh hello", "hello"}} {
var payload proposal_generation.RepairRequest
if err := json.Unmarshal([]byte(client.calls[i].Variables["RepairRequestJSON"]), &payload); err != nil {
@@ -183,7 +194,7 @@ func TestRepairBranchTerminalOutcomes(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
client := &repairClient{t: t, outputs: tc.outputs, err: tc.err}
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: tc.budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: tc.budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
@@ -211,7 +222,7 @@ func TestRunnerAppliesRevisionsInOriginalOrder(t *testing.T) {
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":2,"decision":{"action":"revise","correction":{"id":1,"original_text":"cat","corrected_text":"fox"}}}`,
`{"proposal_ref":3,"decision":{"action":"revise","correction":{"id":1,"original_text":"bird","corrected_text":"owl"}}}`,
}}
module := repairModule{outcomeModule{fakeModule: fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, rejectUntilRepair(0)}}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
module := repairModule{outcomeModule{fakeModule: fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, rejectUntilRepair(0)}}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
return contracts.ProposalResult{Proposals: candidates, SegmentReferences: []int{1, 1, 1}, Exchange: &structuredoutput.Exchange{}}, nil
}}}
runner := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}})
@@ -244,7 +255,7 @@ func TestRepairRevalidatesEarlierApprovalAndRetainsUnrelatedApprovalOnFailure(t
}{{"revision", false}, {"provider failure", true}} {
t.Run(tc.name, func(t *testing.T) {
calls := 0
check := fakeValidator{name: "custom", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
check := fakeValidator{name: "custom", validateF: func(req validators.Request) (validators.Result, error) {
var decisions []validators.Decision
for _, candidate := range req.CandidateProposal {
calls++
@@ -256,7 +267,7 @@ func TestRepairRevalidatesEarlierApprovalAndRetainsUnrelatedApprovalOnFailure(t
if tc.failure {
client.err = &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonProviderFailed, Message: "failed"}
}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat bird"}}}, Validators: []contracts.Validator{check, rejectUntilRepair(0)}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat bird"}}}, Validators: []validators.Validator{check, rejectUntilRepair(0)}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{}}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
@@ -279,7 +290,7 @@ func TestRepairRevalidatesEarlierApprovalAndRetainsUnrelatedApprovalOnFailure(t
func TestRepairInterruptionRetainsAttempt(t *testing.T) {
client := &repairClient{t: t, err: &structuredoutput.Failure{Class: structuredoutput.FailureCanceled, Reason: outcomes.ReasonCanceled, Message: "canceled", Cause: context.Canceled}}
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
@@ -297,7 +308,7 @@ func TestRepairArtifactsAreUnique(t *testing.T) {
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "fox"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}},
}
section := 4
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar_1", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: candidates, SectionIndex: &section, Exchange: &structuredoutput.Exchange{}, Interactions: llm.NewDiagnosticsWriter(t.TempDir())}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar_1", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{rejectUntilRepair()}, SectionEnriched: candidates, SectionIndex: &section, Exchange: &structuredoutput.Exchange{}, Interactions: llm.NewDiagnosticsWriter(t.TempDir())}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
@@ -355,7 +366,7 @@ func TestRepairArtifactFailureDoesNotChangeOutcome(t *testing.T) {
}
sink := llm.NewDiagnosticsWriter(path)
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}, Interactions: sink}
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}, Interactions: sink}
initial, err := validateSectionCandidates(context.Background(), input)
if err != nil {
t.Fatal(err)
@@ -382,8 +393,8 @@ func TestNoEffectAndUnavailableOnlyDoNotStartRepair(t *testing.T) {
}
// rejectUntilRepair supplies a semantic rejection at the external validator seam.
func rejectUntilRepair(preserved ...int) contracts.Validator {
return fakeValidator{name: "editorial_review", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
func rejectUntilRepair(preserved ...int) validators.Validator {
return fakeValidator{name: "editorial_review", validateF: func(req validators.Request) (validators.Result, error) {
var decisions []validators.Decision
for _, p := range req.CandidateProposal {
approved := req.RepairAttempt != nil

View File

@@ -69,13 +69,14 @@ type WorkUnitOutcome struct {
}
type ValidatorDecisionRecord struct {
ValidatorName string `json:"validator_name"`
ProposalIndex int `json:"proposal_index"`
Approved bool `json:"approved"`
Disposition validators.DecisionDisposition `json:"disposition"`
ReasonCode string `json:"reason_code"`
Message string `json:"message"`
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
ExecutionClass validatormetadata.ExecutionClass `json:"-"`
ValidatorName string `json:"validator_name"`
ProposalIndex int `json:"proposal_index"`
Approved bool `json:"approved"`
Disposition validators.DecisionDisposition `json:"disposition"`
ReasonCode string `json:"reason_code"`
Message string `json:"message"`
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
}
type ValidatorRejectedChange struct {
@@ -650,7 +651,7 @@ type validateSectionCandidatesInput struct {
Policy proposals.ReplacementPolicy
WorkingTranscript *schema.Transcript
SectionIndex *int
Validators []contracts.Validator
Validators []validators.Validator
SectionEnriched []proposals.EnrichedCorrectionProposal
ValidationLLMClient contracts.StructuredLLMClient
Interactions structuredoutput.InteractionSink
@@ -673,7 +674,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
eligible := append([]proposals.EnrichedCorrectionProposal(nil), input.SectionEnriched...)
feedback := input.Spec.Settings.LLM.CorrectionRepairAttempts > 0
var blocked, noEffect map[int]bool
evidence := contracts.ValidationRequest{WorkingTranscript: input.WorkingTranscript, Section: input.Section, ReplacementPolicy: input.Policy, IndependentEvidence: feedback}
evidence := validators.Request{WorkingTranscript: input.WorkingTranscript, Section: input.Section, ReplacementPolicy: input.Policy, IndependentEvidence: feedback}
if feedback {
blocked, noEffect = make(map[int]bool), make(map[int]bool)
evidence.Previews = make(map[int]proposals.SegmentPreviewResult)
@@ -686,11 +687,12 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
if err := ctx.Err(); err != nil {
return validateSectionCandidatesResult{decisions: decisions, rejected: rejected, operations: operations, filtered: filtered}, err
}
class := validatormetadata.ClassOf(validator)
submitted := eligible
if feedback {
submitted = feedbackCandidates(input, validator, noEffect, evidence)
submitted = feedbackCandidates(input, validator.Name(), class == validatormetadata.ExecutionClassLLMBacked, noEffect, evidence)
}
vResult, err := validator.Validate(ctx, contracts.ValidationRequest{
vResult, err := validator.Validate(ctx, validators.Request{
StageIndex: &input.Spec.Index, StageID: input.Spec.ID,
WorkingTranscript: input.WorkingTranscript,
Section: evidence.Section, Previews: evidence.Previews,
@@ -728,17 +730,20 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
byIndex[p.ProposalIndex] = p
}
for _, d := range vResult.Decisions {
if feedback && validator.Name() == "original_text_presence" && d.Approved {
preview := evidence.Preview(byIndex[d.ProposalIndex])
if preview.SkipReason == proposals.SkipReasonAmbiguousOriginal {
d.Approved = false
d.Disposition = validators.DispositionRejected
d.ReasonCode = string(proposals.SkipReasonAmbiguousOriginal)
d.Message = "original_text matches multiple locations in the target segment"
}
dispositions := make([]validators.DecisionDisposition, len(vResult.Decisions))
for i, decision := range vResult.Decisions {
disposition, contractErr := decision.EffectiveDisposition()
if contractErr != nil {
return validateSectionCandidatesResult{
decisions: decisions, rejected: rejected, operations: operations, filtered: filtered,
}, fmt.Errorf("validator %q returned inconsistent decisions: %w", validator.Name(), errors.Join(err, contractErr))
}
if feedback && !d.Approved {
dispositions[i] = disposition
}
for i, d := range vResult.Decisions {
disposition := dispositions[i]
approved := disposition == validators.DispositionApproved
if feedback && !approved {
blocked[d.ProposalIndex] = true
if d.ReasonCode == validators.ReasonNoEffect {
noEffect[d.ProposalIndex] = true
@@ -746,18 +751,19 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
}
decisions = append(decisions, ValidatorDecisionRecord{
ValidatorName: validator.Name(),
ExecutionClass: class,
ProposalIndex: d.ProposalIndex,
Approved: d.Approved,
Disposition: d.Disposition,
Approved: approved,
Disposition: disposition,
ReasonCode: d.ReasonCode,
Message: d.Message,
DiagnosticArtifactPath: d.DiagnosticArtifactPath,
})
if d.Approved {
if approved {
nextEligible = append(nextEligible, byIndex[d.ProposalIndex])
continue
}
if d.Disposition != validators.DispositionUnavailable {
if disposition != validators.DispositionUnavailable {
if reason, ok := validators.DefectiveProposalReason(d.ReasonCode); ok {
filtered = append(filtered, outcomes.FilteredProposal{ProposalIndex: d.ProposalIndex, Reason: reason, Count: 1})
}
@@ -771,7 +777,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
TargetSegmentID: p.TargetSegmentID,
OriginalText: p.OriginalText,
CorrectedText: p.CorrectedText,
Disposition: d.Disposition,
Disposition: disposition,
ReasonCode: d.ReasonCode,
Message: d.Message,
})
@@ -882,11 +888,13 @@ func attributeApplyFiltering(sections []chunking.Section, approved []proposals.E
}
}
func reorderValidatorsForPipeline(in []contracts.Validator) ([]contracts.Validator, map[string]int) {
deterministic := make([]contracts.Validator, 0, len(in))
llmBacked := make([]contracts.Validator, 0, len(in))
func reorderValidatorsForPipeline(in []validators.Validator) ([]validators.Validator, map[string]int) {
deterministic := make([]validators.Validator, 0, len(in))
llmBacked := make([]validators.Validator, 0, len(in))
for _, validator := range in {
if validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked {
class := validatormetadata.ClassOf(validator)
validator = validatormetadata.Wrap(validator, class)
if class == validatormetadata.ExecutionClassLLMBacked {
llmBacked = append(llmBacked, validator)
continue
}

View File

@@ -43,7 +43,7 @@ func (f fakeFactory) ModuleForSpec(spec contracts.ModuleRunSpec) (contracts.Tran
type fakeModule struct {
key string
policy proposals.ReplacementPolicy
validators []contracts.Validator
validators []validators.Validator
proposeF func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error)
}
@@ -52,6 +52,33 @@ type outcomeModule struct {
proposeResult func(contracts.ProposalRequest) (contracts.ProposalResult, error)
}
func TestRunnerStopsOnContradictoryValidatorDecision(t *testing.T) {
candidate := proposals.EnrichedCorrectionProposal{
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"},
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 7},
}
contractFailure := errors.New("validator also returned a runtime error")
for _, cause := range []error{nil, contractFailure} {
contradictory := fakeValidator{name: "invalid", validateF: func(validators.Request) (validators.Result, error) {
return validators.Result{Decisions: []validators.Decision{{
ProposalIndex: 7, Approved: true, Disposition: validators.DispositionUnavailable,
}}}, cause
}}
later := fakeValidator{name: "later", validateF: func(validators.Request) (validators.Result, error) {
t.Fatal("candidate advanced past a contradictory decision")
return validators.Result{}, nil
}}
result, err := validateSectionCandidates(context.Background(), validateSectionCandidatesInput{
Validators: []validators.Validator{validators.ProposalShapeValidator{}, contradictory, later},
SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate},
})
if err == nil || !strings.Contains(err.Error(), "contradictory") || (cause != nil && !errors.Is(err, cause)) ||
len(result.approved) != 0 || len(result.decisions) != 1 || result.decisions[0].ValidatorName != "proposal_shape" {
t.Fatalf("contradictory result = %+v (%v)", result, err)
}
}
}
func TestRunnerRecoverableProposalFailureAllowsLaterModule(t *testing.T) {
failed := outcomeModule{fakeModule: fakeModule{key: "first", policy: proposals.ReplacementPolicyRequireUnique}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
return contracts.ProposalResult{Operations: []outcomes.Operation{{Reference: outcomes.Reference{ModuleKey: "first", ModuleInstance: "first", Kind: outcomes.OperationProposal}, FailureReason: outcomes.ReasonTransportFailed}}}, nil
@@ -74,7 +101,7 @@ func TestRunnerRecoverableProposalFailureAllowsLaterModule(t *testing.T) {
func TestRunnerOnlyFullyValidatedCandidatesApplyAndRecoveriesMerge(t *testing.T) {
secondSaw := make([]int, 0)
partial := fakeValidator{name: "partial", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
partial := fakeValidator{name: "partial", validateF: func(req validators.Request) (validators.Result, error) {
if len(req.CandidateProposal) != 2 {
t.Fatalf("expected two candidates after shape filtering, got %d", len(req.CandidateProposal))
}
@@ -86,13 +113,13 @@ func TestRunnerOnlyFullyValidatedCandidatesApplyAndRecoveriesMerge(t *testing.T)
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation}, FailureReason: outcomes.ReasonProviderFailed},
}}, nil
}}
last := fakeValidator{name: "last", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
last := fakeValidator{name: "last", validateF: func(req validators.Request) (validators.Result, error) {
for _, candidate := range req.CandidateProposal {
secondSaw = append(secondSaw, candidate.ProposalIndex)
}
return validators.Result{ValidatorName: "last", Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: true, Disposition: validators.DispositionApproved}}}, nil
}}
module := outcomeModule{fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{validators.ProposalShapeValidator{}, partial, last}}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
module := outcomeModule{fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{validators.ProposalShapeValidator{}, partial, last}}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
return contracts.ProposalResult{Proposals: []proposals.CorrectionProposal{
{TargetSegmentID: 0, OriginalText: "bad", CorrectedText: "x"},
{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"},
@@ -120,10 +147,10 @@ func TestRunnerOnlyFullyValidatedCandidatesApplyAndRecoveriesMerge(t *testing.T)
func TestFatalValidatorNeverReturnsUncheckedCandidates(t *testing.T) {
origin := errors.New("validator setup failed")
proposal := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 7}}
broken := fakeValidator{name: "broken", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
broken := fakeValidator{name: "broken", validateF: func(req validators.Request) (validators.Result, error) {
return validators.Result{Operations: []outcomes.Operation{{Reference: outcomes.Reference{Kind: outcomes.OperationValidation}, FailureReason: outcomes.ReasonSetupFailure}}}, origin
}}
result, err := validateSectionCandidates(context.Background(), validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "m", InstanceName: "m"}, Validators: []contracts.Validator{broken}, SectionEnriched: []proposals.EnrichedCorrectionProposal{proposal}})
result, err := validateSectionCandidates(context.Background(), validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "m", InstanceName: "m"}, Validators: []validators.Validator{broken}, SectionEnriched: []proposals.EnrichedCorrectionProposal{proposal}})
if !errors.Is(err, origin) || len(result.approved) != 0 || len(result.operations) != 1 {
t.Fatalf("fatal result must retain facts without approving candidates: result=%+v err=%v", result, err)
}
@@ -131,7 +158,7 @@ func TestFatalValidatorNeverReturnsUncheckedCandidates(t *testing.T) {
func TestCandidateCaptureIsExplicitAndIncludesFilteredProposals(t *testing.T) {
module := fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique,
validators: []contracts.Validator{validators.ProposalShapeValidator{}},
validators: []validators.Validator{validators.ProposalShapeValidator{}},
proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 0, OriginalText: "bad", CorrectedText: "good"}}, nil
}}
@@ -160,7 +187,7 @@ func (m outcomeModule) Propose(_ context.Context, req contracts.ProposalRequest)
func (m fakeModule) Key() string { return m.key }
func (m fakeModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy }
func (m fakeModule) Validators() []contracts.Validator { return m.validators }
func (m fakeModule) Validators() []validators.Validator { return m.validators }
func (m fakeModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
if m.proposeF == nil {
return contracts.ProposalResult{}, nil
@@ -171,11 +198,11 @@ func (m fakeModule) Propose(ctx context.Context, req contracts.ProposalRequest)
type fakeValidator struct {
name string
validateF func(req contracts.ValidationRequest) (validators.Result, error)
validateF func(req validators.Request) (validators.Result, error)
}
func (v fakeValidator) Name() string { return v.name }
func (v fakeValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (validators.Result, error) {
func (v fakeValidator) Validate(ctx context.Context, req validators.Request) (validators.Result, error) {
_ = ctx
return v.validateF(req)
}
@@ -191,19 +218,19 @@ func (v classifiedFakeValidator) ExecutionClass() validatormetadata.ExecutionCla
func TestReorderValidatorsDeterministicBeforeLLMBacked(t *testing.T) {
llm := classifiedFakeValidator{
fakeValidator: fakeValidator{name: "llm", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
fakeValidator: fakeValidator{name: "llm", validateF: func(req validators.Request) (validators.Result, error) {
return validators.Result{ValidatorName: "llm"}, nil
}},
class: validatormetadata.ExecutionClassLLMBacked,
}
deterministic := classifiedFakeValidator{
fakeValidator: fakeValidator{name: "deterministic", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
fakeValidator: fakeValidator{name: "deterministic", validateF: func(req validators.Request) (validators.Result, error) {
return validators.Result{ValidatorName: "deterministic"}, nil
}},
class: validatormetadata.ExecutionClassDeterministic,
}
ordered, _ := reorderValidatorsForPipeline([]contracts.Validator{llm, deterministic})
ordered, _ := reorderValidatorsForPipeline([]validators.Validator{llm, deterministic})
if len(ordered) != 2 {
t.Fatalf("expected 2 validators, got %d", len(ordered))
}
@@ -213,17 +240,17 @@ func TestReorderValidatorsDeterministicBeforeLLMBacked(t *testing.T) {
}
func TestReorderValidatorsDefaultsUnclassifiedToDeterministic(t *testing.T) {
unclassified := fakeValidator{name: "plain", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
unclassified := fakeValidator{name: "plain", validateF: func(req validators.Request) (validators.Result, error) {
return validators.Result{ValidatorName: "plain"}, nil
}}
llm := classifiedFakeValidator{
fakeValidator: fakeValidator{name: "llm", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
fakeValidator: fakeValidator{name: "llm", validateF: func(req validators.Request) (validators.Result, error) {
return validators.Result{ValidatorName: "llm"}, nil
}},
class: validatormetadata.ExecutionClassLLMBacked,
}
ordered, _ := reorderValidatorsForPipeline([]contracts.Validator{llm, unclassified})
ordered, _ := reorderValidatorsForPipeline([]validators.Validator{llm, unclassified})
if len(ordered) != 2 {
t.Fatalf("expected 2 validators, got %d", len(ordered))
}
@@ -553,7 +580,7 @@ func TestRunnerValidationStartsBeforeAllSectionProposalsComplete(t *testing.T) {
validator := fakeValidator{
name: "capture",
validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
validateF: func(req validators.Request) (validators.Result, error) {
if len(req.CandidateProposal) == 1 && req.CandidateProposal[0].TargetSegmentID == 1 {
select {
case validatorStarted <- struct{}{}:
@@ -577,7 +604,7 @@ func TestRunnerValidationStartsBeforeAllSectionProposalsComplete(t *testing.T) {
"m": fakeModule{
key: "m",
policy: proposals.ReplacementPolicyRequireUnique,
validators: []contracts.Validator{validator},
validators: []validators.Validator{validator},
proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
seg := req.WorkingTranscript.Segments[0]
if req.Section != nil && req.Section.Index == 1 {
@@ -630,7 +657,7 @@ func (c *captureContractStructuredClient) CompleteStructured(ctx context.Context
if target, ok := out.(*validators.LLMValidationResponse); ok {
*target = validators.LLMValidationResponse{
Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
{CorrectionIndex: "1", Approved: true, Reason: "ok"},
},
}
}
@@ -662,7 +689,7 @@ func (c *stageAwareStructuredClient) CompleteStructured(ctx context.Context, req
target := out.(*validators.LLMValidationResponse)
*target = validators.LLMValidationResponse{
Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
{CorrectionIndex: "1", Approved: true, Reason: "ok"},
},
}
return contracts.StructuredCompletionResponse{}, nil
@@ -734,7 +761,7 @@ func TestRunnerRepeatedModuleInstanceNames(t *testing.T) {
func TestRunnerValidatorApprovedProposalApplied(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}}
allowAll := fakeValidator{name: "allow", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
allowAll := fakeValidator{name: "allow", validateF: func(req validators.Request) (validators.Result, error) {
decisions := make([]validators.Decision, len(req.CandidateProposal))
for i, p := range req.CandidateProposal {
decisions[i] = validators.Decision{ProposalIndex: p.ProposalIndex, Approved: true, ReasonCode: validators.ReasonApproved, Message: "approved"}
@@ -745,7 +772,7 @@ func TestRunnerValidatorApprovedProposalApplied(t *testing.T) {
"m": fakeModule{
key: "m",
policy: proposals.ReplacementPolicyRequireUnique,
validators: []contracts.Validator{allowAll},
validators: []validators.Validator{allowAll},
proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil
},
@@ -766,7 +793,7 @@ func TestRunnerValidatorApprovedProposalApplied(t *testing.T) {
func TestRunnerValidatorRejectedProposalNotApplied(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}}
rejectAll := fakeValidator{name: "reject", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
rejectAll := fakeValidator{name: "reject", validateF: func(req validators.Request) (validators.Result, error) {
decisions := make([]validators.Decision, len(req.CandidateProposal))
for i, p := range req.CandidateProposal {
decisions[i] = validators.Decision{ProposalIndex: p.ProposalIndex, Approved: false, ReasonCode: validators.ReasonNoEffect, Message: "rejected"}
@@ -774,7 +801,7 @@ func TestRunnerValidatorRejectedProposalNotApplied(t *testing.T) {
return validators.Result{ValidatorName: "reject", Decisions: decisions}, nil
}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{rejectAll}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{rejectAll}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil
}},
}})
@@ -792,7 +819,7 @@ func TestRunnerValidatorRejectedProposalNotApplied(t *testing.T) {
func TestRunnerMultipleValidatorsRunInOrderAndFilterSurvivors(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "one two"}}}
first := fakeValidator{name: "first", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
first := fakeValidator{name: "first", validateF: func(req validators.Request) (validators.Result, error) {
if len(req.CandidateProposal) != 2 {
t.Fatalf("expected first validator to see 2 candidates, got %d", len(req.CandidateProposal))
}
@@ -804,14 +831,14 @@ func TestRunnerMultipleValidatorsRunInOrderAndFilterSurvivors(t *testing.T) {
},
}, nil
}}
second := fakeValidator{name: "second", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
second := fakeValidator{name: "second", validateF: func(req validators.Request) (validators.Result, error) {
if len(req.CandidateProposal) != 1 {
t.Fatalf("expected second validator to see only survivors, got %d", len(req.CandidateProposal))
}
return validators.Result{ValidatorName: "second", Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: true, ReasonCode: validators.ReasonApproved, Message: "ok"}}}, nil
}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{first, second}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{first, second}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{
{TargetSegmentID: 1, OriginalText: "one", CorrectedText: "ONE"},
{TargetSegmentID: 1, OriginalText: "two", CorrectedText: "TWO"},
@@ -833,11 +860,11 @@ func TestRunnerValidatorCardinalityErrorStopsPipelineWithPartialProgress(t *test
good := fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil
}}
badValidator := fakeValidator{name: "bad", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
badValidator := fakeValidator{name: "bad", validateF: func(req validators.Request) (validators.Result, error) {
// Missing one decision triggers cardinality error.
return validators.Result{ValidatorName: "bad", Decisions: nil}, nil
}}
bad := fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{badValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
bad := fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{badValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}}, nil
}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"m1": good, "m2": bad}})
@@ -856,11 +883,11 @@ func TestRunnerValidatorCardinalityErrorStopsPipelineWithPartialProgress(t *test
func TestRunnerApplicationSkipAfterValidatorApprovalReported(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "word word"}}}
allow := fakeValidator{name: "allow", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
allow := fakeValidator{name: "allow", validateF: func(req validators.Request) (validators.Result, error) {
return validators.Result{ValidatorName: "allow", Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: true, ReasonCode: validators.ReasonApproved, Message: "ok"}}}, nil
}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{allow}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{allow}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "word", CorrectedText: "term"}}, nil
}},
}})
@@ -880,7 +907,7 @@ func TestRunnerApplyConflictAfterTwoInitiallyValidDeletions(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "a b"}}}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": outcomeModule{
fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NonEmptyResultingSegmentValidator{}}},
fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NonEmptyResultingSegmentValidator{}}},
proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
return contracts.ProposalResult{
Proposals: []proposals.CorrectionProposal{
@@ -945,14 +972,14 @@ func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contr
}
func TestRunnerLLMValidatorApprovalApplied(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}}}
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}}}
llmValidator, err := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
if err != nil {
t.Fatalf("NewLLMBackedValidator: %v", err)
}
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil
}},
}})
@@ -971,10 +998,10 @@ func TestRunnerLLMValidatorApprovalApplied(t *testing.T) {
}
func TestRunnerLLMValidatorRejectionPreventsApplication(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: false, Reason: "reject"}}}}}
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: false, Reason: "reject"}}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil
}},
}})
@@ -996,13 +1023,13 @@ func TestRunnerLLMValidatorRejectionPreventsApplication(t *testing.T) {
}
func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Reason: "bad index"}}}}}
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "99", Approved: true, Reason: "bad index"}}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m1": fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil
}},
"m2": fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
"m2": fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}}, nil
}},
}})
@@ -1028,13 +1055,13 @@ func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress(
func TestRunnerReconciledSurvivorsCompleteLaterValidation(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
{CorrectionIndex: 3, Approved: true, Reason: "ok"},
{CorrectionIndex: 99, Approved: true, Reason: "unexpected"},
{CorrectionIndex: "1", Approved: true, Reason: "ok"},
{CorrectionIndex: "3", Approved: true, Reason: "ok"},
{CorrectionIndex: "99", Approved: true, Reason: "unexpected"},
}}}}
first, _ := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
var laterIndices []int
later := fakeValidator{name: "later_review", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
later := fakeValidator{name: "later_review", validateF: func(req validators.Request) (validators.Result, error) {
decisions := make([]validators.Decision, 0, len(req.CandidateProposal))
for _, candidate := range req.CandidateProposal {
laterIndices = append(laterIndices, candidate.ProposalIndex)
@@ -1047,7 +1074,7 @@ func TestRunnerReconciledSurvivorsCompleteLaterValidation(t *testing.T) {
}
return validators.Result{ValidatorName: "later_review", Decisions: decisions}, nil
}}
module := outcomeModule{fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{first, validatormetadata.Wrap(later, validatormetadata.ExecutionClassLLMBacked)}}, proposeResult: func(contracts.ProposalRequest) (contracts.ProposalResult, error) {
module := outcomeModule{fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{first, validatormetadata.Wrap(later, validatormetadata.ExecutionClassLLMBacked)}}, proposeResult: func(contracts.ProposalRequest) (contracts.ProposalResult, error) {
return contracts.ProposalResult{Proposals: []proposals.CorrectionProposal{
{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"},
{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"},
@@ -1077,7 +1104,7 @@ func TestRunnerLLMValidatorMissingDecisionRejectsBatch(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil
}},
}})
@@ -1094,12 +1121,12 @@ func TestRunnerLLMValidatorMissingDecisionRejectsBatch(t *testing.T) {
func TestRunnerLLMValidatorDuplicateDecisionRejectsBatch(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
{CorrectionIndex: 1, Approved: false, Reason: "dup"},
{CorrectionIndex: "1", Approved: true, Reason: "ok"},
{CorrectionIndex: "1", Approved: false, Reason: "dup"},
}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil
}},
}})
@@ -1120,8 +1147,8 @@ func (lenEstimator) EstimateTokens(text string) int { return len(text) }
func TestRunnerLLMValidatorBatching(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}},
}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
llmValidator.SetTokenEstimator(lenEstimator{})
@@ -1129,7 +1156,7 @@ func TestRunnerLLMValidatorBatching(t *testing.T) {
cfg := runnerTestConfig()
cfg.Validation.MaxPromptTokens = 260
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyReplaceAll, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyReplaceAll, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{
{TargetSegmentID: 1, OriginalText: "alpha", CorrectedText: strings.Repeat("B", 40)},
{TargetSegmentID: 1, OriginalText: "gamma", CorrectedText: strings.Repeat("D", 40)},
@@ -1154,12 +1181,12 @@ func TestRunnerLLMValidatorBatching(t *testing.T) {
}
func TestRunnerLLMValidatorDiagnosticsWritten(t *testing.T) {
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "supported by context"}}}}}
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "supported by context"}}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
cfg := runnerTestConfig()
diagDir := t.TempDir()
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil
}},
}})
@@ -1208,7 +1235,7 @@ type proposalGenerationModule struct {
func (m proposalGenerationModule) Key() string { return m.key }
func (m proposalGenerationModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy }
func (m proposalGenerationModule) Validators() []contracts.Validator { return nil }
func (m proposalGenerationModule) Validators() []validators.Validator { return nil }
func (m proposalGenerationModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
result, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
ModuleKey: req.RunSpec.ModuleKey,
@@ -1294,10 +1321,10 @@ func TestFatalPartialDecisionsRejectUnknownAndDuplicateIdentity(t *testing.T) {
{{ProposalIndex: 8, Approved: true}},
{{ProposalIndex: 7, Approved: true}, {ProposalIndex: 7, Approved: true}},
} {
v := fakeValidator{name: "broken", validateF: func(contracts.ValidationRequest) (validators.Result, error) {
v := fakeValidator{name: "broken", validateF: func(validators.Request) (validators.Result, error) {
return validators.Result{Decisions: decisions}, origin
}}
result, err := validateSectionCandidates(context.Background(), validateSectionCandidatesInput{Validators: []contracts.Validator{v}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}})
result, err := validateSectionCandidates(context.Background(), validateSectionCandidatesInput{Validators: []validators.Validator{v}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}})
if !errors.Is(err, origin) || len(result.decisions) != 0 || len(result.approved) != 0 {
t.Fatalf("invalid partial evidence accepted or origin lost: %+v %v", result, err)
}

View File

@@ -37,6 +37,7 @@ func (v OriginalTextPresenceValidator) Name() string { return "original_text_pre
func (v OriginalTextPresenceValidator) Validate(_ context.Context, req Request) (Result, error) {
section := req.sectionLookup()
req.Section = section
decisions := make([]Decision, 0, len(req.CandidateProposal))
for _, c := range req.CandidateProposal {
@@ -49,6 +50,10 @@ func (v OriginalTextPresenceValidator) Validate(_ context.Context, req Request)
decisions = append(decisions, rejection(c.ProposalIndex, ReasonMissingOriginalText, "original_text was not found in current segment text"))
continue
}
if req.IndependentEvidence && req.Preview(c).SkipReason == proposals.SkipReasonAmbiguousOriginal {
decisions = append(decisions, rejection(c.ProposalIndex, string(proposals.SkipReasonAmbiguousOriginal), "original_text matches multiple locations in the target segment"))
continue
}
decisions = append(decisions, approval(c.ProposalIndex))
}
if err := EnforceDecisionCardinality(req.CandidateProposal, decisions); err != nil {

View File

@@ -25,15 +25,15 @@ func TestValidationContextCarriesAdjacentEvidenceAndRelevantDefinitions(t *testi
{ID: "lyra", Name: "Lyra", Description: "The host"},
{ID: "unrelated", Name: "Unrelated", Description: "Not mentioned"},
}
payload, immediate := BuildLLMValidationRequest("spoken_form_plausibility", LLMValidatorTypeSpokenFormPlausibility, req)
if len(immediate) != 0 || len(payload.Items) != 1 {
payload, immediate := buildValidationItems(req)
if len(immediate) != 0 || len(payload) != 1 {
t.Fatalf("payload: %+v immediate: %+v", payload, immediate)
}
item := payload.Items[0]
item := payload[0]
if len(item.NeighboringSegments) != 2 || item.NeighboringSegments[0].Position != "preceding" || item.NeighboringSegments[1].Position != "following" || len(item.GlossaryEntries) != 2 || item.GlossaryEntries[0].Name != "Jesters" || item.GlossaryEntries[1].Name != "Lyra" {
t.Fatalf("context: %+v", item)
}
variables, err := BuildValidationVariables(payload.Items, "")
variables, err := BuildValidationVariables(payload, "")
if err != nil {
t.Fatal(err)
}
@@ -41,7 +41,7 @@ func TestValidationContextCarriesAdjacentEvidenceAndRelevantDefinitions(t *testi
if err := json.Unmarshal([]byte(variables["PayloadJSON"]), &rendered); err != nil {
t.Fatal(err)
}
if strings.Contains(variables["PayloadJSON"], `"segment_id":`) || rendered[0].CorrectionIndex != 1 || payload.Items[0].CorrectionIndex != 7 {
if strings.Contains(variables["PayloadJSON"], `"segment_id":`) || rendered[0].CorrectionIndex != 1 || payload[0].CorrectionIndex != 7 {
t.Fatalf("leaked internal identity or mutated request: %s", variables["PayloadJSON"])
}
if rendered[0].Speaker != "Guest" || len(rendered[0].NeighboringSegments) != 2 || rendered[0].NeighboringSegments[0].Speaker != "Host" || rendered[0].GlossaryEntries[0].Description != "A traveling troupe" {
@@ -51,15 +51,15 @@ func TestValidationContextCarriesAdjacentEvidenceAndRelevantDefinitions(t *testi
req.WorkingTranscript.Segments = req.WorkingTranscript.Segments[1:2]
req.WorkingTranscript.Segments[0].Speaker = ""
req.Glossary = nil
isolated, _ := BuildLLMValidationRequest("editorial_review", LLMValidatorTypeEditorialReview, req)
if isolated.Items[0].Speaker != "" || len(isolated.Items[0].NeighboringSegments) != 0 || len(isolated.Items[0].GlossaryEntries) != 0 {
isolated, _ := buildValidationItems(req)
if isolated[0].Speaker != "" || len(isolated[0].NeighboringSegments) != 0 || len(isolated[0].GlossaryEntries) != 0 {
t.Fatalf("unexpected external context: %+v", isolated)
}
}
func TestValidationContextFitsBudgetWithoutDiscardingCandidate(t *testing.T) {
estimator := chunking.NewSimpleTokenEstimator()
base := LLMValidationItem{CorrectionIndex: 7, SegmentID: 1, OriginalText: "teh", CorrectedText: "the", OriginalSegmentText: "teh cat", CorrectedSegmentText: "the cat"}
base := LLMValidationItem{CorrectionIndex: 7, OriginalText: "teh", CorrectedText: "the", OriginalSegmentText: "teh cat", CorrectedSegmentText: "the cat"}
required, err := estimateBatchTokens(estimator, []LLMValidationItem{base})
if err != nil {
t.Fatal(err)
@@ -89,12 +89,12 @@ func TestValidationContextSelectsGlossaryForEachReplacement(t *testing.T) {
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(7, "gestures", "Jesters"), mk(11, "gestures", "Actors")})
req.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Those gestures are back."}}}
req.Glossary = []glossary.Entry{{ID: "actors", Name: "Actors"}, {ID: "jesters", Name: "Jesters"}}
payload, immediate := BuildLLMValidationRequest("editorial_review", LLMValidatorTypeEditorialReview, req)
if len(immediate) != 0 || len(payload.Items) != 2 {
payload, immediate := buildValidationItems(req)
if len(immediate) != 0 || len(payload) != 2 {
t.Fatalf("payload: %+v immediate: %+v", payload, immediate)
}
for i, name := range []string{"Jesters", "Actors"} {
entries := payload.Items[i].GlossaryEntries
entries := payload[i].GlossaryEntries
if len(entries) != 1 || entries[0].Name != name {
t.Fatalf("candidate %d context: %+v", i, entries)
}

View File

@@ -15,6 +15,7 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
ck "go.maximumdirect.net/configkit"
)
@@ -50,28 +51,7 @@ func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) {
}
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
})
dir := t.TempDir()
main := filepath.Join(dir, "config.yml")
if err := os.WriteFile(main, []byte("version: 3\n"), 0600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte("profiles:\n - {id: base, endpoint: 'https://example.invalid/v1', model: local-model, api_key_required: false}\n"), 0600); err != nil {
t.Fatal(err)
}
loaded, err := config.Load(ck.Input{Source: ck.File(main)}, "")
if err != nil {
t.Fatal(err)
}
runtime, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
if err != nil {
t.Fatal(err)
}
settings := loaded.Config.LLM
settings.TransientRetries, settings.RepairAttempts = 0, 1
client, err := runtime.NewRoleClient("validation", "base", settings, llm.CallHooks{})
if err != nil {
t.Fatal(err)
}
client := testValidationClient(t, transport)
validator, err := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
if err != nil {
t.Fatal(err)
@@ -91,3 +71,98 @@ func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) {
})
}
}
// The transport supplies provider responses; PromptKit performs structured
// validation and Audita decodes the same response through the role client.
func testValidationClient(t *testing.T, transport http.RoundTripper) structuredoutput.Client {
t.Helper()
dir := t.TempDir()
main := filepath.Join(dir, "config.yml")
if err := os.WriteFile(main, []byte("version: 3\n"), 0600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte("profiles:\n - {id: base, endpoint: 'https://example.invalid/v1', model: local-model, api_key_required: false}\n"), 0600); err != nil {
t.Fatal(err)
}
loaded, err := config.Load(ck.Input{Source: ck.File(main)}, "")
if err != nil {
t.Fatal(err)
}
runtime, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
if err != nil {
t.Fatal(err)
}
settings := loaded.Config.LLM
settings.TransientRetries, settings.RepairAttempts = 0, 1
client, err := runtime.NewRoleClient("validation", "base", settings, llm.CallHooks{})
if err != nil {
t.Fatal(err)
}
return client
}
func TestValidationDecodesExactReferencesBeforeReconciliation(t *testing.T) {
for _, tc := range []struct {
name, response string
indices []int
want []DecisionDisposition
reasons []string
failure outcomes.ReasonCode
}{
{
name: "usable decision beside extra",
response: `{"validations":[{"correction_index":1.00e0,"approved":true,"reason":"accept"},{"correction_index":9223372036854775808,"approved":true,"reason":"extra"}]}`,
indices: []int{7},
want: []DecisionDisposition{DispositionApproved},
reasons: []string{ReasonApproved},
},
{
name: "usable siblings beside duplicate and omission",
response: `{"validations":[{"correction_index":1e0,"approved":true,"reason":"accept"},{"correction_index":2,"approved":true,"reason":"duplicate"},{"correction_index":20e-1,"approved":false,"reason":"duplicate"},{"correction_index":4.000,"approved":false,"reason":"reject"},{"correction_index":9223372036854775808,"approved":true,"reason":"extra"}]}`,
indices: []int{7, 11, 14, 23},
want: []DecisionDisposition{DispositionApproved, DispositionUnavailable, DispositionUnavailable, DispositionRejected},
reasons: []string{ReasonApproved, ReasonDuplicateValidationDecision, ReasonMissingValidationDecision, "llm_rejected"},
failure: outcomes.ReasonValidationDecisionsInvalid,
},
} {
t.Run(tc.name, func(t *testing.T) {
calls := 0
client := testValidationClient(t, validationTransport(func(req *http.Request) (*http.Response, error) {
calls++
content, _ := json.Marshal(tc.response)
body := fmt.Sprintf(`{"choices":[{"message":{"content":%s}}]}`, content)
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: req}, nil
}))
var candidates []proposals.EnrichedCorrectionProposal
for _, index := range tc.indices {
candidates = append(candidates, mk(index, "gestures", "Jesters"))
}
validator, err := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
if err != nil {
t.Fatal(err)
}
sink := &captureValidationDiagnosticsWriter{}
req := makeReq(candidates)
req.LLMClient = client
req.Interactions = sink
result, err := validator.Validate(context.Background(), req)
if err != nil || calls != 1 || len(result.Decisions) != len(tc.want) || len(result.Operations) != 1 {
t.Fatalf("result=%+v calls=%d error=%v", result, calls, err)
}
for i, decision := range result.Decisions {
if decision.ProposalIndex != tc.indices[i] || decision.Disposition != tc.want[i] || decision.ReasonCode != tc.reasons[i] {
t.Fatalf("decision %d: %+v", i, decision)
}
}
operation := result.Operations[0]
if operation.FailureReason != tc.failure || (tc.failure == "" && (len(operation.RecoveryReasons) != 1 || operation.RecoveryReasons[0] != outcomes.RecoveryValidationDecisionsFiltered)) {
t.Fatalf("operation: %+v", operation)
}
payload := sink.lastResponsePayload.(map[string]any)
diagnostics := payload["reconciliation"].(validationReconciliation)
if len(diagnostics.IgnoredIndices) != 1 || diagnostics.IgnoredIndices[0] != "9223372036854775808" || payload["final_content"] != tc.response {
t.Fatalf("reference evidence changed: %+v", payload)
}
})
}
}

View File

@@ -1,6 +1,8 @@
package validators
import (
"encoding/json"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
)
@@ -21,7 +23,6 @@ type StructuredLLMClient = structuredoutput.Client
// BuildValidationVariables substitutes a one-based reference for each batch.
type LLMValidationItem struct {
CorrectionIndex int `json:"correction_index"`
SegmentID int `json:"-"`
Speaker string `json:"speaker"`
OriginalText string `json:"original_text"`
CorrectedText string `json:"corrected_text"`
@@ -32,21 +33,11 @@ type LLMValidationItem struct {
GlossaryEntries []glossary.Entry `json:"glossary_entries,omitempty"`
}
// LLMValidationRequest is the canonical request model for LLM-backed validators.
type LLMValidationRequest struct {
ValidatorName string `json:"validator_name"`
ValidatorType LLMValidatorType `json:"validator_type"`
ModuleKey string `json:"module_key"`
ModuleInstance string `json:"module_instance"`
ReplacementPolicy string `json:"replacement_policy"`
Items []LLMValidationItem `json:"corrections"`
}
// LLMValidationDecision contains a request-local correction_index, not a proposal index.
type LLMValidationDecision struct {
CorrectionIndex int `json:"correction_index"`
Approved bool `json:"approved"`
Reason string `json:"reason"`
CorrectionIndex json.Number `json:"correction_index"`
Approved bool `json:"approved"`
Reason string `json:"reason"`
}
type LLMValidationResponse struct {

View File

@@ -48,9 +48,9 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
}
failed := &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonTransportFailed, Message: "model transport failed"}
client := &scriptedValidationClient{replies: []validationReply{
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}},
{completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 1}}, err: failed},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}},
}}
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
validator.SetTokenEstimator(perItemEstimator{})
@@ -73,8 +73,8 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
survivors := []proposals.EnrichedCorrectionProposal{candidates[0], candidates[2]}
nextClient := &scriptedValidationClient{replies: []validationReply{
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}},
}}
nextValidator, _ := NewLLMBackedValidator("meaning_reversal_review", LLMValidatorTypeMeaningReversal)
nextValidator.SetTokenEstimator(perItemEstimator{})
@@ -101,7 +101,7 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
func TestValidationDomainInvalidAfterRepairIsAnError(t *testing.T) {
client := &scriptedValidationClient{replies: []validationReply{{
response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Reason: "ok"}}},
response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "99", Approved: true, Reason: "ok"}}},
completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 2, StructuralRepairs: 1}},
}}}
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
@@ -116,8 +116,8 @@ func TestValidationDomainInvalidAfterRepairIsAnError(t *testing.T) {
func TestValidationExtraDecisionsMergeActualRecoveries(t *testing.T) {
client := &scriptedValidationClient{replies: []validationReply{{
response: LLMValidationResponse{Validations: []LLMValidationDecision{
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
{CorrectionIndex: 99, Approved: true, Reason: "extra"},
{CorrectionIndex: "1", Approved: true, Reason: "ok"},
{CorrectionIndex: "99", Approved: true, Reason: "extra"},
}},
completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 3, TransientRetries: 1, StructuralRepairs: 1}},
}}}
@@ -137,9 +137,9 @@ func TestValidationExtraDecisionsMergeActualRecoveries(t *testing.T) {
func TestValidationFatalFailureKeepsEarlierFactsAndStops(t *testing.T) {
cause := errors.New("broken embedded schema")
client := &scriptedValidationClient{replies: []validationReply{
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}},
{err: &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "model setup failed", Cause: cause}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}},
}}
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
validator.SetTokenEstimator(perItemEstimator{})

View File

@@ -2,6 +2,7 @@ package validators
import (
"context"
"encoding/json"
"errors"
"fmt"
"sort"
@@ -9,6 +10,7 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/jsonnumber"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/stagename"
@@ -58,8 +60,8 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
if req.LLMClient == nil {
return Result{}, fmt.Errorf("LLM-backed validator %q requires a structured LLM client", v.name)
}
validationReq, immediate := BuildLLMValidationRequest(v.name, v.validatorType, req)
if len(validationReq.Items) == 0 {
items, immediate := buildValidationItems(req)
if len(items) == 0 {
all := append([]Decision(nil), immediate...)
sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex })
if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil {
@@ -73,7 +75,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
maxTokens = req.Settings.Validation.MaxPromptTokens
}
batches, oversized, err := prepareValidationBatches(validationReq.Items, maxTokens, v.estimator)
batches, oversized, err := prepareValidationBatches(items, maxTokens, v.estimator)
operations := make([]outcomes.Operation, 0)
llmDecisions := make([]Decision, 0)
partialResult := func() Result {
@@ -239,7 +241,7 @@ func errPayload(err error) any {
return map[string]any{"error": err.Error()}
}
func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorType, req Request) (LLMValidationRequest, []Decision) {
func buildValidationItems(req Request) ([]LLMValidationItem, []Decision) {
items := make([]LLMValidationItem, 0, len(req.CandidateProposal))
immediate := make([]Decision, 0)
req.Section = req.sectionLookup()
@@ -268,7 +270,6 @@ func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorT
neighbors, glossaryEntries := glossary.validationContext(segments, position, preview.CorrectedSegmentText)
items = append(items, LLMValidationItem{
CorrectionIndex: p.ProposalIndex,
SegmentID: p.TargetSegmentID,
Speaker: seg.Speaker,
OriginalText: p.OriginalText,
CorrectedText: p.CorrectedText,
@@ -280,31 +281,25 @@ func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorT
})
}
return LLMValidationRequest{
ValidatorName: validatorName,
ValidatorType: validatorType,
ModuleKey: req.ModuleKey,
ModuleInstance: req.ModuleInstance,
ReplacementPolicy: string(req.ReplacementPolicy),
Items: items,
}, immediate
return items, immediate
}
type validationReconciliation struct {
Decisions []Decision `json:"decisions"`
UnavailableIndices []int `json:"unavailable_indices,omitempty"`
IgnoredIndices []int `json:"ignored_indices,omitempty"`
Decisions []Decision `json:"decisions"`
UnavailableIndices []int `json:"unavailable_indices,omitempty"`
IgnoredIndices []json.Number `json:"ignored_indices,omitempty"`
}
func reconcileLLMResponse(items []LLMValidationItem, response LLMValidationResponse) validationReconciliation {
result := validationReconciliation{Decisions: make([]Decision, 0, len(items))}
seen := make(map[int][]LLMValidationDecision, len(items))
for _, d := range response.Validations {
if d.CorrectionIndex < 1 || d.CorrectionIndex > len(items) {
index, ok := jsonnumber.PositiveIndex(d.CorrectionIndex.String(), len(items))
if !ok {
result.IgnoredIndices = append(result.IgnoredIndices, d.CorrectionIndex)
continue
}
seen[d.CorrectionIndex] = append(seen[d.CorrectionIndex], d)
seen[index] = append(seen[index], d)
}
for i, item := range items {

View File

@@ -4,6 +4,8 @@ import (
"context"
"encoding/json"
"errors"
"slices"
"strconv"
"strings"
"testing"
@@ -74,7 +76,7 @@ func mk(index int, orig, corr string) proposals.EnrichedCorrectionProposal {
}
}
func TestBuildLLMValidationRequestPreservesLiteralDeletionAndSpacing(t *testing.T) {
func TestValidationPayloadPreservesLiteralDeletionAndSpacing(t *testing.T) {
for _, tc := range []struct {
name, segment, original, replacement, result string
}{
@@ -84,27 +86,25 @@ func TestBuildLLMValidationRequestPreservesLiteralDeletionAndSpacing(t *testing.
t.Run(tc.name, func(t *testing.T) {
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(7, tc.original, tc.replacement)})
req.WorkingTranscript.Segments[0].Text = tc.segment
payload, immediate := BuildLLMValidationRequest("editorial_review", LLMValidatorTypeEditorialReview, req)
if len(immediate) != 0 || len(payload.Items) != 1 {
payload, immediate := buildValidationItems(req)
if len(immediate) != 0 || len(payload) != 1 {
t.Fatalf("expected previewable request: %+v, %+v", payload, immediate)
}
raw, err := json.Marshal(payload)
variables, err := BuildValidationVariables(payload, "")
if err != nil {
t.Fatal(err)
}
var decoded struct {
ValidatorName string `json:"validator_name"`
Corrections []map[string]any `json:"corrections"`
}
raw := []byte(variables["PayloadJSON"])
var decoded []map[string]any
if err := json.Unmarshal(raw, &decoded); err != nil {
t.Fatal(err)
}
if decoded.ValidatorName != "editorial_review" || len(decoded.Corrections) != 1 {
t.Fatalf("wrong validator identity or item count: %s", raw)
if len(decoded) != 1 {
t.Fatalf("wrong item count: %s", raw)
}
item := decoded.Corrections[0]
item := decoded[0]
value, present := item["corrected_text"]
if !present || value != tc.replacement || item["original_text"] != tc.original || item["original_segment_text"] != tc.segment || item["corrected_segment_text"] != tc.result || item["correction_index"] != float64(7) {
if !present || value != tc.replacement || item["original_text"] != tc.original || item["original_segment_text"] != tc.segment || item["corrected_segment_text"] != tc.result || item["correction_index"] != float64(1) {
t.Fatalf("literal spans or full segment text lost: %s", raw)
}
})
@@ -148,8 +148,8 @@ func TestChunkLLMValidationItemsMultipleBatchesStableNoDropNoDup(t *testing.T) {
func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
{CorrectionIndex: 1, Approved: true, Reason: ""},
{CorrectionIndex: 2, Approved: false, Reason: "bad"},
{CorrectionIndex: "1", Approved: true, Reason: ""},
{CorrectionIndex: "2", Approved: false, Reason: "bad"},
}}}}
v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
if err != nil {
@@ -203,10 +203,10 @@ func TestLLMBackedValidatorProviderMalformedEnvelopePropagates(t *testing.T) {
func TestLLMBackedValidatorReconcilesDecisionIdentities(t *testing.T) {
approve := func(index int) LLMValidationDecision {
return LLMValidationDecision{CorrectionIndex: index, Approved: true, Reason: "accepted"}
return LLMValidationDecision{CorrectionIndex: json.Number(strconv.Itoa(index)), Approved: true, Reason: "accepted"}
}
reject := func(index int) LLMValidationDecision {
return LLMValidationDecision{CorrectionIndex: index, Reason: "editorial concern"}
return LLMValidationDecision{CorrectionIndex: json.Number(strconv.Itoa(index)), Reason: "editorial concern"}
}
type expectedDecision struct {
index int
@@ -218,13 +218,13 @@ func TestLLMBackedValidatorReconcilesDecisionIdentities(t *testing.T) {
entries []LLMValidationDecision
want []expectedDecision
unavailable []int
ignored []int
ignored []json.Number
}{
{"internal index is not a local reference", []LLMValidationDecision{approve(7), approve(2), reject(3)}, []expectedDecision{{7, false, ReasonMissingValidationDecision}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, []int{7}, []int{7}},
{"missing and extra", []LLMValidationDecision{approve(1), reject(3), approve(99)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonMissingValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, []int{99}},
{"internal index is not a local reference", []LLMValidationDecision{approve(7), approve(2), reject(3)}, []expectedDecision{{7, false, ReasonMissingValidationDecision}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, []int{7}, []json.Number{"7"}},
{"missing and extra", []LLMValidationDecision{approve(1), reject(3), approve(99)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonMissingValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, []json.Number{"99"}},
{"conflicting duplicate", []LLMValidationDecision{approve(1), approve(2), reject(2), reject(3)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonDuplicateValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
{"identical duplicate", []LLMValidationDecision{approve(1), approve(2), approve(2), reject(3)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonDuplicateValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
{"extra only", []LLMValidationDecision{approve(99), reject(3), approve(1), approve(99), approve(2)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, []int{99, 99}},
{"extra only", []LLMValidationDecision{approve(99), reject(3), approve(1), approve(99), approve(2)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, []json.Number{"99", "99"}},
{"empty", nil, []expectedDecision{{7, false, ReasonMissingValidationDecision}, {11, false, ReasonMissingValidationDecision}, {18, false, ReasonMissingValidationDecision}}, []int{7, 11, 18}, nil},
{"reordered", []LLMValidationDecision{reject(3), approve(2), approve(1)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, nil},
} {
@@ -264,7 +264,7 @@ func TestLLMBackedValidatorReconcilesDecisionIdentities(t *testing.T) {
t.Fatalf("supplied model decisions lost from diagnostic: %+v", decoded)
}
reconciliation := payload["reconciliation"].(validationReconciliation)
if !equalIndices(reconciliation.UnavailableIndices, tc.unavailable) || !equalIndices(reconciliation.IgnoredIndices, tc.ignored) || len(reconciliation.Decisions) != len(tc.want) {
if !equalIndices(reconciliation.UnavailableIndices, tc.unavailable) || !slices.Equal(reconciliation.IgnoredIndices, tc.ignored) || len(reconciliation.Decisions) != len(tc.want) {
t.Fatalf("diagnostic reconciliation=%+v", reconciliation)
}
})
@@ -285,7 +285,7 @@ func equalIndices(a, b []int) bool {
func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
{CorrectionIndex: "1", Approved: true, Reason: "ok"},
}}}}
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
huge := strings.Repeat("gestures ", 200)
@@ -331,7 +331,7 @@ func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *tes
func TestLLMBackedValidatorDiagnosticsIncludeSchemaMetadata(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
{CorrectionIndex: "1", Approved: true, Reason: "ok"},
}}}}
writer := &captureValidationDiagnosticsWriter{}
v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
@@ -370,7 +370,7 @@ func TestEveryValidationTypeSelectsEmbeddedPromptAndVariables(t *testing.T) {
}
for _, tc := range cases {
t.Run(string(tc.kind), func(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}}}
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}}}
validator, err := NewLLMBackedValidator(string(tc.kind), tc.kind)
if err != nil {
t.Fatal(err)
@@ -395,7 +395,7 @@ func TestEveryValidationTypeSelectsEmbeddedPromptAndVariables(t *testing.T) {
func TestValidationDiagnosticFailureDoesNotChangeDecision(t *testing.T) {
cause := errors.New("diagnostic storage unavailable")
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}}}
req.LLMClient = &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}}}
sink := &captureValidationDiagnosticsWriter{err: cause}
req.Interactions = sink
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)

View File

@@ -62,6 +62,25 @@ type Decision struct {
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
}
// EffectiveDisposition resolves an omitted disposition from the declared
// approval and detects inconsistent explicit decisions.
func (d Decision) EffectiveDisposition() (DecisionDisposition, error) {
switch d.Disposition {
case "":
if d.Approved {
return DispositionApproved, nil
}
return DispositionRejected, nil
case DispositionApproved, DispositionRejected, DispositionUnavailable:
if d.Approved == (d.Disposition == DispositionApproved) {
return d.Disposition, nil
}
return "", fmt.Errorf("contradictory approval and disposition %q for proposal %d", d.Disposition, d.ProposalIndex)
default:
return "", fmt.Errorf("invalid decision disposition %q for proposal %d", d.Disposition, d.ProposalIndex)
}
}
// DecisionDisposition separates normal editorial rejection from a candidate
// excluded because its model operation could not provide a trustworthy decision.
type DecisionDisposition string
@@ -80,7 +99,10 @@ type Result struct {
Operations []outcomes.Operation `json:"operations,omitempty"`
}
// Validator is the deterministic runtime validator interface.
// Validator evaluates one section's candidates. An instance may be called
// concurrently across sections and may keep synchronized state. Request data
// is borrowed for the call; inputs must be copied before modification or
// retention. Request.Preview may populate the caller's pass-local cache.
type Validator interface {
Name() string
Validate(ctx context.Context, req Request) (Result, error)

View File

@@ -54,3 +54,21 @@ func (r Request) Preview(candidate proposals.EnrichedCorrectionProposal) proposa
}
return preview
}
// AcceptsIndependentCandidate decides which evidence can be submitted when
// repair needs findings from each applicable validator.
func (r Request) AcceptsIndependentCandidate(validatorName string, modelDriven bool, candidate proposals.EnrichedCorrectionProposal) bool {
hasSpan := candidate.OriginalText != ""
switch validatorName {
case "proposal_shape":
return true
case "no_effect":
return hasSpan
case "original_text_presence":
return hasSpan && candidate.TargetSegmentID > 0
case "protected_terms", "non_empty_resulting_segment":
return r.Preview(candidate).HasResult()
default:
return !modelDriven || r.Preview(candidate).Applicable
}
}

View File

@@ -2,6 +2,7 @@ package glossary
import (
"context"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
@@ -11,7 +12,7 @@ import (
)
type Module struct {
validators []contracts.Validator
validators []frameworkvalidators.Validator
}
func New() (*Module, error) {
@@ -33,8 +34,8 @@ func (m *Module) ReplacementPolicy() proposals.ReplacementPolicy {
return proposals.ReplacementPolicyReplaceAll
}
func (m *Module) Validators() []contracts.Validator {
return append([]contracts.Validator(nil), m.validators...)
func (m *Module) Validators() []frameworkvalidators.Validator {
return append([]frameworkvalidators.Validator(nil), m.validators...)
}
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {

View File

@@ -2,6 +2,7 @@ package grammar
import (
"context"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
@@ -11,7 +12,7 @@ import (
)
type Module struct {
validators []contracts.Validator
validators []frameworkvalidators.Validator
}
func New() (*Module, error) {
@@ -33,8 +34,8 @@ func (m *Module) ReplacementPolicy() proposals.ReplacementPolicy {
return proposals.ReplacementPolicyRequireUnique
}
func (m *Module) Validators() []contracts.Validator {
return append([]contracts.Validator(nil), m.validators...)
func (m *Module) Validators() []frameworkvalidators.Validator {
return append([]frameworkvalidators.Validator(nil), m.validators...)
}
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {

View File

@@ -2,6 +2,7 @@ package homophones
import (
"context"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
@@ -11,7 +12,7 @@ import (
)
type Module struct {
validators []contracts.Validator
validators []frameworkvalidators.Validator
}
func New() (*Module, error) {
@@ -33,8 +34,8 @@ func (m *Module) ReplacementPolicy() proposals.ReplacementPolicy {
return proposals.ReplacementPolicyRequireUnique
}
func (m *Module) Validators() []contracts.Validator {
return append([]contracts.Validator(nil), m.validators...)
func (m *Module) Validators() []frameworkvalidators.Validator {
return append([]frameworkvalidators.Validator(nil), m.validators...)
}
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {

View File

@@ -154,7 +154,7 @@ func TestHomophonesProtectedGlossaryTermBehaviorRejectsUnsafeCorrection(t *testi
if err != nil {
t.Fatalf("New error: %v", err)
}
var protectedValidator contracts.Validator
var protectedValidator validators.Validator
for _, v := range module.Validators() {
if v.Name() == "protected_terms" {
protectedValidator = v

View File

@@ -2,6 +2,7 @@ package spoken_word
import (
"context"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
@@ -11,7 +12,7 @@ import (
)
type Module struct {
validators []contracts.Validator
validators []frameworkvalidators.Validator
}
func New() (*Module, error) {
@@ -33,8 +34,8 @@ func (m *Module) ReplacementPolicy() proposals.ReplacementPolicy {
return proposals.ReplacementPolicyRequireUnique
}
func (m *Module) Validators() []contracts.Validator {
return append([]contracts.Validator(nil), m.validators...)
func (m *Module) Validators() []frameworkvalidators.Validator {
return append([]frameworkvalidators.Validator(nil), m.validators...)
}
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {

View File

@@ -154,7 +154,7 @@ func TestSpokenWordProtectedGlossaryTermBehaviorRejectsUnsafeCorrection(t *testi
if err != nil {
t.Fatalf("New error: %v", err)
}
var protectedValidator contracts.Validator
var protectedValidator validators.Validator
for _, v := range module.Validators() {
if v.Name() == "protected_terms" {
protectedValidator = v

View File

@@ -2,10 +2,10 @@ package validators
import (
"fmt"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
"strings"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
)
var builtInChains = map[string][]string{
@@ -58,7 +58,7 @@ func BuiltInChainKeys(moduleKey string) ([]string, error) {
return out, nil
}
func ResolveBuiltInChain(moduleKey string, registry *Registry) ([]contracts.Validator, error) {
func ResolveBuiltInChain(moduleKey string, registry *Registry) ([]frameworkvalidators.Validator, error) {
moduleKey = strings.TrimSpace(moduleKey)
keys, err := BuiltInChainKeys(moduleKey)
if err != nil {
@@ -68,7 +68,7 @@ func ResolveBuiltInChain(moduleKey string, registry *Registry) ([]contracts.Vali
registry = NewBuiltInRegistry()
}
out := make([]contracts.Validator, 0, len(keys))
out := make([]frameworkvalidators.Validator, 0, len(keys))
for _, key := range keys {
v, buildErr := registry.MustBuild(key)
if buildErr != nil {

View File

@@ -1,14 +1,13 @@
package editorial_review
import (
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
)
const Key = "editorial_review"
func New() (contracts.Validator, error) {
func New() (frameworkvalidators.Validator, error) {
v, err := frameworkvalidators.NewLLMBackedValidator(Key, frameworkvalidators.LLMValidatorTypeEditorialReview)
if err != nil {
return nil, err

View File

@@ -1,14 +1,13 @@
package meaning_reversal_review
import (
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
)
const Key = "meaning_reversal_review"
func New() (contracts.Validator, error) {
func New() (frameworkvalidators.Validator, error) {
v, err := frameworkvalidators.NewLLMBackedValidator(Key, frameworkvalidators.LLMValidatorTypeMeaningReversal)
if err != nil {
return nil, err

View File

@@ -3,7 +3,7 @@ package metadata
import (
"strings"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
)
type ExecutionClass string
@@ -37,11 +37,11 @@ var executionClassByKey = map[string]ExecutionClass{
}
type ClassifiedValidator interface {
contracts.Validator
frameworkvalidators.Validator
ExecutionClass() ExecutionClass
}
func ClassOf(v contracts.Validator) ExecutionClass {
func ClassOf(v frameworkvalidators.Validator) ExecutionClass {
if v == nil {
return ExecutionClassDeterministic
}
@@ -69,7 +69,7 @@ func ClassForKey(key string) ExecutionClass {
return class
}
func Wrap(v contracts.Validator, class ExecutionClass) contracts.Validator {
func Wrap(v frameworkvalidators.Validator, class ExecutionClass) frameworkvalidators.Validator {
if v == nil {
return nil
}
@@ -83,7 +83,7 @@ func Wrap(v contracts.Validator, class ExecutionClass) contracts.Validator {
}
type wrappedValidator struct {
contracts.Validator
frameworkvalidators.Validator
class ExecutionClass
}

View File

@@ -4,7 +4,6 @@ import (
"context"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
)
@@ -12,7 +11,7 @@ type unclassifiedValidator struct{}
func (u unclassifiedValidator) Name() string { return "unclassified" }
func (u unclassifiedValidator) Validate(_ context.Context, _ contracts.ValidationRequest) (frameworkvalidators.Result, error) {
func (u unclassifiedValidator) Validate(_ context.Context, _ frameworkvalidators.Request) (frameworkvalidators.Result, error) {
return frameworkvalidators.Result{ValidatorName: u.Name(), Decisions: nil}, nil
}
@@ -22,7 +21,7 @@ type namedUnclassifiedValidator struct {
func (n namedUnclassifiedValidator) Name() string { return n.name }
func (n namedUnclassifiedValidator) Validate(_ context.Context, _ contracts.ValidationRequest) (frameworkvalidators.Result, error) {
func (n namedUnclassifiedValidator) Validate(_ context.Context, _ frameworkvalidators.Request) (frameworkvalidators.Result, error) {
return frameworkvalidators.Result{ValidatorName: n.Name(), Decisions: nil}, nil
}

View File

@@ -3,7 +3,7 @@ package validators_test
import (
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
"gitea.maximumdirect.net/eric/audita/v3/internal/modules/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/modules/grammar"
"gitea.maximumdirect.net/eric/audita/v3/internal/modules/homophones"
@@ -20,30 +20,30 @@ func TestProductionModulesUseRegisteredValidatorKeys(t *testing.T) {
cases := []struct {
name string
build func() ([]contracts.Validator, error)
build func() ([]frameworkvalidators.Validator, error)
}{
{name: "glossary", build: func() ([]contracts.Validator, error) {
{name: "glossary", build: func() ([]frameworkvalidators.Validator, error) {
m, err := glossary.New()
if err != nil {
return nil, err
}
return m.Validators(), nil
}},
{name: "homophones", build: func() ([]contracts.Validator, error) {
{name: "homophones", build: func() ([]frameworkvalidators.Validator, error) {
m, err := homophones.New()
if err != nil {
return nil, err
}
return m.Validators(), nil
}},
{name: "spoken_word", build: func() ([]contracts.Validator, error) {
{name: "spoken_word", build: func() ([]frameworkvalidators.Validator, error) {
m, err := spoken_word.New()
if err != nil {
return nil, err
}
return m.Validators(), nil
}},
{name: "grammar", build: func() ([]contracts.Validator, error) {
{name: "grammar", build: func() ([]frameworkvalidators.Validator, error) {
m, err := grammar.New()
if err != nil {
return nil, err

View File

@@ -1,13 +1,12 @@
package no_effect
import (
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
)
const Key = "no_effect"
func New() (contracts.Validator, error) {
func New() (frameworkvalidators.Validator, error) {
return validatormetadata.Wrap(frameworkvalidators.NoEffectValidator{}, validatormetadata.ExecutionClassDeterministic), nil
}

View File

@@ -1,13 +1,12 @@
package non_empty_resulting_segment
import (
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
)
const Key = "non_empty_resulting_segment"
func New() (contracts.Validator, error) {
func New() (frameworkvalidators.Validator, error) {
return validatormetadata.Wrap(frameworkvalidators.NonEmptyResultingSegmentValidator{}, validatormetadata.ExecutionClassDeterministic), nil
}

View File

@@ -1,13 +1,12 @@
package original_text_presence
import (
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
)
const Key = "original_text_presence"
func New() (contracts.Validator, error) {
func New() (frameworkvalidators.Validator, error) {
return validatormetadata.Wrap(frameworkvalidators.OriginalTextPresenceValidator{}, validatormetadata.ExecutionClassDeterministic), nil
}

View File

@@ -1,13 +1,12 @@
package proposal_shape
import (
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
)
const Key = "proposal_shape"
func New() (contracts.Validator, error) {
func New() (frameworkvalidators.Validator, error) {
return validatormetadata.Wrap(frameworkvalidators.ProposalShapeValidator{}, validatormetadata.ExecutionClassDeterministic), nil
}

View File

@@ -3,14 +3,13 @@ package protected_terms
import (
"context"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
)
const Key = "protected_terms"
func New() (contracts.Validator, error) {
func New() (frameworkvalidators.Validator, error) {
return validatormetadata.Wrap(validator{}, validatormetadata.ExecutionClassDeterministic), nil
}
@@ -18,6 +17,6 @@ type validator struct{}
func (v validator) Name() string { return Key }
func (v validator) Validate(_ context.Context, req contracts.ValidationRequest) (frameworkvalidators.Result, error) {
func (v validator) Validate(_ context.Context, req frameworkvalidators.Request) (frameworkvalidators.Result, error) {
return frameworkvalidators.ValidateProtectedTerms(req)
}

View File

@@ -6,7 +6,6 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
@@ -50,7 +49,7 @@ func TestNewUsesGeneralBehaviorForGlossaryModule(t *testing.T) {
t.Fatalf("New: %v", err)
}
req := contracts.ValidationRequest{
req := frameworkvalidators.Request{
ModuleKey: "glossary",
Glossary: tinyGlossary(),
CandidateProposal: []proposals.EnrichedCorrectionProposal{
@@ -78,7 +77,7 @@ func TestNewRejectsProtectedTermChangeOutsideGlossaryStage(t *testing.T) {
t.Fatalf("New: %v", err)
}
req := contracts.ValidationRequest{
req := frameworkvalidators.Request{
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Jesters"}}},
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
ModuleKey: "homophones",
@@ -115,7 +114,7 @@ func TestNewGlossaryStageRejectsProtectedTermChange(t *testing.T) {
t.Fatalf("expected deterministic class, got %q", got)
}
req := contracts.ValidationRequest{
req := frameworkvalidators.Request{
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Jesters"}}},
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
ModuleKey: "glossary",
@@ -146,7 +145,7 @@ func TestNewGlossaryStageAllowsProtectedTermInsertion(t *testing.T) {
t.Fatalf("New: %v", err)
}
req := contracts.ValidationRequest{
req := frameworkvalidators.Request{
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "gestures"}}},
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
ModuleKey: "glossary",

View File

@@ -2,9 +2,9 @@ package validators
import (
"fmt"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
"strings"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/validators/editorial_review"
"gitea.maximumdirect.net/eric/audita/v3/internal/validators/meaning_reversal_review"
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
@@ -30,7 +30,7 @@ const (
type BuiltInValidatorDefinition struct {
Key string
Build func() (contracts.Validator, error)
Build func() (frameworkvalidators.Validator, error)
}
type Registry struct {
@@ -64,7 +64,7 @@ func (r *Registry) Lookup(key string) (BuiltInValidatorDefinition, bool) {
return def, ok
}
func (r *Registry) MustBuild(key string) (contracts.Validator, error) {
func (r *Registry) MustBuild(key string) (frameworkvalidators.Validator, error) {
if r == nil {
return nil, fmt.Errorf("validator registry is nil")
}

View File

@@ -2,12 +2,12 @@ package validators
import (
"context"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/validators/editorial_review"
"gitea.maximumdirect.net/eric/audita/v3/internal/validators/meaning_reversal_review"
@@ -50,7 +50,7 @@ func TestBuiltInValidatorPackagesConstruct(t *testing.T) {
type validatorCtor struct {
name string
key string
build func() (contracts.Validator, error)
build func() (frameworkvalidators.Validator, error)
wantClass validatormetadata.ExecutionClass
}
cases := []validatorCtor{
@@ -117,7 +117,7 @@ func TestRegistryProtectedTermsUsesNonGlossaryStageBehavior(t *testing.T) {
t.Fatalf("must build protected_terms: %v", err)
}
req := contracts.ValidationRequest{
req := frameworkvalidators.Request{
ModuleKey: "glossary",
Glossary: []glossary.Entry{{Collection: "party", ID: "jesters", Name: "Jesters", Category: "faction", Description: "Protected"}},
CandidateProposal: []proposals.EnrichedCorrectionProposal{
@@ -152,7 +152,7 @@ func TestResolveBuiltInChainUsesGlossaryStageProtectedTermsBehavior(t *testing.T
t.Fatalf("resolve glossary chain: %v", err)
}
var v contracts.Validator
var v frameworkvalidators.Validator
for _, candidate := range chain {
if candidate.Name() == KeyProtectedTerms {
v = candidate
@@ -163,7 +163,7 @@ func TestResolveBuiltInChainUsesGlossaryStageProtectedTermsBehavior(t *testing.T
t.Fatalf("expected protected_terms validator in glossary chain")
}
req := contracts.ValidationRequest{
req := frameworkvalidators.Request{
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Jesters"}}},
ReplacementPolicy: proposals.ReplacementPolicyReplaceAll,
ModuleKey: "glossary",

View File

@@ -1,14 +1,13 @@
package spoken_form_plausibility
import (
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
)
const Key = "spoken_form_plausibility"
func New() (contracts.Validator, error) {
func New() (frameworkvalidators.Validator, error) {
v, err := frameworkvalidators.NewLLMBackedValidator(Key, frameworkvalidators.LLMValidatorTypeSpokenFormPlausibility)
if err != nil {
return nil, err