Validate internal decisions and reconcile exact model references
This commit is contained in:
@@ -53,9 +53,17 @@ assessment is unavailable; an unavailable assessment alone does not. Revised
|
||||
candidates run the applicable checks again. A zero repair budget retains the
|
||||
original shortlist flow.
|
||||
|
||||
Decision cardinality is enforced inside Audita: each candidate proposal must
|
||||
receive exactly one outcome per validator it reaches. The runner still rejects
|
||||
duplicate, missing, or unknown identities in an internal validator result.
|
||||
The shared validator port lives in `internal/framework/validators`. Instances
|
||||
may run concurrently across sections and can keep synchronized state. Request
|
||||
data is borrowed for the duration of the call. Copy data before modifying or
|
||||
retaining it.
|
||||
|
||||
The runner checks decision identity and disposition before allowing a candidate
|
||||
to advance. A normal return supplies one decision per submitted candidate. A
|
||||
fatal return may provide a checked subset as evidence. Contradictory approval
|
||||
and disposition are a fatal contract failure. The recorded disposition governs
|
||||
approval, and each record carries the effective execution class, including
|
||||
instance overrides, for initial validation and revalidation.
|
||||
|
||||
## LLM Validator Batching
|
||||
|
||||
@@ -89,8 +97,11 @@ Reconciliation makes no semantic retry or extra model call; guided recovery rema
|
||||
[deferred](../roadmap/future.md#domain-aware-response-recovery).
|
||||
|
||||
Debug request metadata records `proposal_indices` in local-reference order
|
||||
(reference 1 is the first entry). Raw decoded responses and ignored indices use
|
||||
local references; reconciled decisions and unavailable indices use internal
|
||||
(reference 1 is the first entry). Decoding preserves the supplied numeric
|
||||
tokens, including integral decimal or exponent spellings. Reconciliation
|
||||
checks the batch range before converting to a local integer, so an extra
|
||||
reference beyond the native integer range can be ignored. Raw decoded responses
|
||||
and ignored indices use local references. Reconciled decisions use internal
|
||||
proposal indices. Batch token estimates use the same locally numbered payload
|
||||
as prompt encoding.
|
||||
|
||||
|
||||
@@ -355,6 +355,13 @@ beside an out-of-range reference. Cover equivalent numeric spellings,
|
||||
contradictory internal results, and class override from execution through the
|
||||
record supplied to Stage 17. Update [validator internals](../internal/validators.md).
|
||||
|
||||
**Complete.** Validators share one port and applicability owner. The runner
|
||||
rejects contradictory decisions and records the effective class on initial and
|
||||
repair validation evidence. Controlled provider responses verified exact
|
||||
numeric decoding, equivalent spellings, duplicate and missing decisions, and
|
||||
a usable decision alongside a reference above the native integer range. The
|
||||
CLI build and full offline suite passed.
|
||||
|
||||
## Stage 10 — Reduce validation context work
|
||||
|
||||
Audit #36.
|
||||
|
||||
82
internal/core/jsonnumber/integer.go
Normal file
82
internal/core/jsonnumber/integer.go
Normal file
@@ -0,0 +1,82 @@
|
||||
// Package jsonnumber converts exact integral JSON tokens without float rounding.
|
||||
package jsonnumber
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Int accepts lexical integers and equivalent decimal and exponent spellings
|
||||
// in the native integer range. value must be a valid JSON number token.
|
||||
func Int(value string) (int, bool) {
|
||||
if integer, err := strconv.Atoi(value); err == nil {
|
||||
return integer, true
|
||||
}
|
||||
digits, ok := integralDecimal(value)
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
integer, err := strconv.Atoi(digits)
|
||||
return integer, err == nil
|
||||
}
|
||||
|
||||
// PositiveIndex accepts only references in [1, maximum]. Comparison with the
|
||||
// requested range precedes conversion to the local integer index.
|
||||
func PositiveIndex(value string, maximum int) (int, bool) {
|
||||
if strings.ContainsAny(value, ".eE") {
|
||||
var ok bool
|
||||
if value, ok = integralDecimal(value); !ok {
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
if maximum < 1 || value == "" || value[0] < '1' || value[0] > '9' {
|
||||
return 0, false
|
||||
}
|
||||
limit := strconv.Itoa(maximum)
|
||||
if len(value) > len(limit) || (len(value) == len(limit) && value > limit) {
|
||||
return 0, false
|
||||
}
|
||||
index, err := strconv.Atoi(value)
|
||||
return index, err == nil
|
||||
}
|
||||
|
||||
func integralDecimal(value string) (string, bool) {
|
||||
if value == "" || (value[0] != '-' && (value[0] < '0' || value[0] > '9')) || !strings.ContainsAny(value, ".eE") {
|
||||
return "", false
|
||||
}
|
||||
digits, negative := strings.CutPrefix(value, "-")
|
||||
var exponentText string
|
||||
if i := strings.IndexAny(digits, "eE"); i >= 0 {
|
||||
exponentText, digits = digits[i+1:], digits[:i]
|
||||
}
|
||||
whole, fraction, _ := strings.Cut(digits, ".")
|
||||
digits = strings.TrimLeft(whole+fraction, "0")
|
||||
if digits == "" {
|
||||
return "0", true
|
||||
}
|
||||
var exponent int
|
||||
if exponentText != "" {
|
||||
var err error
|
||||
exponent, err = strconv.Atoi(exponentText)
|
||||
if err != nil || exponent < -len(digits) || exponent > len(fraction)+19 {
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
shift := exponent - len(fraction)
|
||||
if shift < 0 {
|
||||
// A negative shift is integral only if every discarded digit is zero.
|
||||
if -shift > len(digits)-len(strings.TrimRight(digits, "0")) {
|
||||
return "", false
|
||||
}
|
||||
digits = digits[:len(digits)+shift]
|
||||
} else {
|
||||
if len(digits)+shift > 19 {
|
||||
return "", false
|
||||
}
|
||||
digits += strings.Repeat("0", shift)
|
||||
}
|
||||
if negative {
|
||||
digits = "-" + digits
|
||||
}
|
||||
return digits, true
|
||||
}
|
||||
@@ -6,8 +6,8 @@ import (
|
||||
"fmt"
|
||||
"math"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/jsonnumber"
|
||||
)
|
||||
|
||||
type SourceSegment struct {
|
||||
@@ -114,60 +114,13 @@ func extractSegments(raw []byte) (json.RawMessage, error) {
|
||||
type sourceID int
|
||||
|
||||
func (id *sourceID) UnmarshalJSON(raw []byte) error {
|
||||
spelling := string(raw)
|
||||
if value, err := strconv.Atoi(spelling); err == nil {
|
||||
*id = sourceID(value)
|
||||
return nil
|
||||
}
|
||||
if value, ok := integralDecimal(spelling); ok {
|
||||
if value, ok := jsonnumber.Int(string(raw)); ok {
|
||||
*id = sourceID(value)
|
||||
return nil
|
||||
}
|
||||
return json.Unmarshal(raw, (*int)(id))
|
||||
}
|
||||
|
||||
func integralDecimal(value string) (int, bool) {
|
||||
if value == "" || (value[0] != '-' && (value[0] < '0' || value[0] > '9')) || !strings.ContainsAny(value, ".eE") {
|
||||
return 0, false
|
||||
}
|
||||
digits, negative := strings.CutPrefix(value, "-")
|
||||
var exponentText string
|
||||
if i := strings.IndexAny(digits, "eE"); i >= 0 {
|
||||
exponentText, digits = digits[i+1:], digits[:i]
|
||||
}
|
||||
whole, fraction, _ := strings.Cut(digits, ".")
|
||||
digits = strings.TrimLeft(whole+fraction, "0")
|
||||
if digits == "" {
|
||||
return 0, true
|
||||
}
|
||||
var exponent int
|
||||
if exponentText != "" {
|
||||
var err error
|
||||
exponent, err = strconv.Atoi(exponentText)
|
||||
if err != nil || exponent < -len(digits) || exponent > len(fraction)+19 {
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
shift := exponent - len(fraction)
|
||||
if shift < 0 {
|
||||
// A negative shift is integral only if every discarded digit is zero.
|
||||
if -shift > len(digits)-len(strings.TrimRight(digits, "0")) {
|
||||
return 0, false
|
||||
}
|
||||
digits = digits[:len(digits)+shift]
|
||||
} else {
|
||||
if len(digits)+shift > 19 {
|
||||
return 0, false
|
||||
}
|
||||
digits += strings.Repeat("0", shift)
|
||||
}
|
||||
if negative {
|
||||
digits = "-" + digits
|
||||
}
|
||||
integer, err := strconv.Atoi(digits)
|
||||
return integer, err == nil
|
||||
}
|
||||
|
||||
func validateSourceSegments(segments []SourceSegment) error {
|
||||
for i, s := range segments {
|
||||
segLabel := segmentLabel(i, s.ID)
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
||||
)
|
||||
|
||||
// testChunkProposalHarness is a test-only helper that composes existing
|
||||
@@ -77,7 +78,7 @@ func (m deterministicFakeModule) ReplacementPolicy() proposals.ReplacementPolicy
|
||||
return m.replacementPolicy
|
||||
}
|
||||
|
||||
func (m deterministicFakeModule) Validators() []Validator { return nil }
|
||||
func (m deterministicFakeModule) Validators() []validators.Validator { return nil }
|
||||
|
||||
func (m deterministicFakeModule) Propose(ctx context.Context, req ProposalRequest) (ProposalResult, error) {
|
||||
_ = ctx
|
||||
|
||||
@@ -22,16 +22,10 @@ type StructuredLLMClient = structuredoutput.Client
|
||||
type TranscriptModule interface {
|
||||
Key() string
|
||||
ReplacementPolicy() proposals.ReplacementPolicy
|
||||
Validators() []Validator
|
||||
Validators() []validators.Validator
|
||||
Propose(ctx context.Context, req ProposalRequest) (ProposalResult, error)
|
||||
}
|
||||
|
||||
// Validator evaluates candidate proposals and returns one decision per proposal index.
|
||||
type Validator interface {
|
||||
Name() string
|
||||
Validate(ctx context.Context, req ValidationRequest) (validators.Result, error)
|
||||
}
|
||||
|
||||
type StructuredCompletionRequest = structuredoutput.Request
|
||||
type StructuredCompletionResponse = structuredoutput.Response
|
||||
|
||||
@@ -98,9 +92,6 @@ type ProposalResult struct {
|
||||
Operations []outcomes.Operation `json:"operations,omitempty"`
|
||||
}
|
||||
|
||||
// ValidationRequest is the input to validator execution.
|
||||
type ValidationRequest = validators.Request
|
||||
|
||||
// ResolveModuleRunSpecs derives isolated consumer views from resolved settings.
|
||||
func ResolveModuleRunSpecs(cfg *config.Config) []ModuleRunSpec {
|
||||
if cfg == nil {
|
||||
|
||||
@@ -30,7 +30,7 @@ type fakeValidator struct{}
|
||||
|
||||
func (f *fakeValidator) Name() string { return "fake-validator" }
|
||||
|
||||
func (f *fakeValidator) Validate(ctx context.Context, req ValidationRequest) (validators.Result, error) {
|
||||
func (f *fakeValidator) Validate(ctx context.Context, req validators.Request) (validators.Result, error) {
|
||||
_ = ctx
|
||||
decisions := make([]validators.Decision, len(req.CandidateProposal))
|
||||
for i, proposal := range req.CandidateProposal {
|
||||
@@ -47,8 +47,8 @@ func (f *fakeModule) ReplacementPolicy() proposals.ReplacementPolicy {
|
||||
return proposals.ReplacementPolicyRequireUnique
|
||||
}
|
||||
|
||||
func (f *fakeModule) Validators() []Validator {
|
||||
return []Validator{&fakeValidator{}}
|
||||
func (f *fakeModule) Validators() []validators.Validator {
|
||||
return []validators.Validator{&fakeValidator{}}
|
||||
}
|
||||
|
||||
func (f *fakeModule) Propose(ctx context.Context, req ProposalRequest) (ProposalResult, error) {
|
||||
@@ -63,7 +63,7 @@ func (f *fakeModule) Propose(ctx context.Context, req ProposalRequest) (Proposal
|
||||
|
||||
func TestInterfaceContractsCompileWithFakes(t *testing.T) {
|
||||
var _ StructuredLLMClient = (*fakeLLMClient)(nil)
|
||||
var _ Validator = (*fakeValidator)(nil)
|
||||
var _ validators.Validator = (*fakeValidator)(nil)
|
||||
var _ TranscriptModule = (*fakeModule)(nil)
|
||||
|
||||
module := &fakeModule{}
|
||||
|
||||
@@ -3,6 +3,7 @@ package modules
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -20,7 +21,7 @@ func (m noopModule) Key() string { return m.key }
|
||||
func (m noopModule) ReplacementPolicy() proposals.ReplacementPolicy {
|
||||
return proposals.ReplacementPolicyRequireUnique
|
||||
}
|
||||
func (m noopModule) Validators() []contracts.Validator { return nil }
|
||||
func (m noopModule) Validators() []frameworkvalidators.Validator { return nil }
|
||||
func (m noopModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
_ = ctx
|
||||
_ = req
|
||||
|
||||
@@ -2,7 +2,9 @@ package processreport
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -16,13 +18,15 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
||||
)
|
||||
|
||||
type partialModule struct{ validator contracts.Validator }
|
||||
type partialModule struct{ validator validators.Validator }
|
||||
|
||||
func (m partialModule) Key() string { return "grammar" }
|
||||
func (m partialModule) ReplacementPolicy() proposals.ReplacementPolicy {
|
||||
return proposals.ReplacementPolicyRequireUnique
|
||||
}
|
||||
func (m partialModule) Validators() []contracts.Validator { return []contracts.Validator{m.validator} }
|
||||
func (m partialModule) Validators() []validators.Validator {
|
||||
return []validators.Validator{m.validator}
|
||||
}
|
||||
func (m partialModule) ModuleForSpec(contracts.ModuleRunSpec) (contracts.TranscriptModule, error) {
|
||||
return m, nil
|
||||
}
|
||||
@@ -55,7 +59,7 @@ func (c *interruptedValidationClient) CompleteStructured(_ context.Context, req
|
||||
*target.(*validators.LLMValidationResponse) = validators.LLMValidationResponse{}
|
||||
return structuredoutput.Response{}, nil
|
||||
}
|
||||
*target.(*validators.LLMValidationResponse) = validators.LLMValidationResponse{Validations: []validators.LLMValidationDecision{{CorrectionIndex: index, Approved: index == 1, Reason: "completed assessment"}}}
|
||||
*target.(*validators.LLMValidationResponse) = validators.LLMValidationResponse{Validations: []validators.LLMValidationDecision{{CorrectionIndex: json.Number(strconv.Itoa(index)), Approved: index == 1, Reason: "completed assessment"}}}
|
||||
return structuredoutput.Response{}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -104,7 +104,7 @@ func TestValidatorFailureCancelsActiveAndPendingProviderWork(t *testing.T) {
|
||||
}, &out)
|
||||
return validators.Result{}, err
|
||||
}}
|
||||
module := fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{validator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
module := fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{validator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "hello", CorrectedText: "Hello"}}, nil
|
||||
}}
|
||||
sections := []chunking.Section{}
|
||||
|
||||
@@ -1,48 +1,19 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
||||
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
|
||||
)
|
||||
|
||||
// feedbackCandidates submits only candidates for which the validator has usable evidence.
|
||||
func feedbackCandidates(input validateSectionCandidatesInput, validator contracts.Validator, noEffect map[int]bool, evidence contracts.ValidationRequest) []proposals.EnrichedCorrectionProposal {
|
||||
func feedbackCandidates(input validateSectionCandidatesInput, validatorName string, modelDriven bool, noEffect map[int]bool, evidence validators.Request) []proposals.EnrichedCorrectionProposal {
|
||||
var result []proposals.EnrichedCorrectionProposal
|
||||
for _, candidate := range input.SectionEnriched {
|
||||
if noEffect[candidate.ProposalIndex] {
|
||||
continue
|
||||
}
|
||||
usableSpan := candidate.OriginalText != ""
|
||||
target := candidate.TargetSegmentID > 0
|
||||
var applicable bool
|
||||
switch validator.Name() {
|
||||
case "proposal_shape":
|
||||
applicable = true
|
||||
case "no_effect":
|
||||
applicable = usableSpan
|
||||
case "original_text_presence":
|
||||
applicable = usableSpan && target
|
||||
case "protected_terms":
|
||||
applicable = usableSpan
|
||||
if input.Spec.ModuleKey == modulecatalog.KeyGlossary || input.Spec.ModuleKey == modulecatalog.KeySpokenWord {
|
||||
preview := evidence.Preview(candidate)
|
||||
applicable = preview.Applicable || preview.SkipReason == proposals.SkipReasonEmptyResultingText
|
||||
}
|
||||
case "non_empty_resulting_segment":
|
||||
preview := evidence.Preview(candidate)
|
||||
applicable = preview.Applicable || preview.SkipReason == proposals.SkipReasonEmptyResultingText
|
||||
default:
|
||||
if validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked {
|
||||
applicable = evidence.Preview(candidate).Applicable
|
||||
} else {
|
||||
applicable = true
|
||||
}
|
||||
}
|
||||
if applicable {
|
||||
if evidence.AcceptsIndependentCandidate(validatorName, modelDriven, candidate) {
|
||||
result = append(result, candidate)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,7 +25,7 @@ func TestFeedbackCollectsIndependentFindingsAndUsableSemanticInput(t *testing.T)
|
||||
return proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: target, OriginalText: original, CorrectedText: replacement}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index}}
|
||||
}
|
||||
var seen []int
|
||||
semantic := validatormetadata.Wrap(fakeValidator{name: "semantic", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
semantic := validatormetadata.Wrap(fakeValidator{name: "semantic", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
var decisions []validators.Decision
|
||||
for _, p := range req.CandidateProposal {
|
||||
seen = append(seen, p.ProposalIndex)
|
||||
@@ -33,7 +33,7 @@ func TestFeedbackCollectsIndependentFindingsAndUsableSemanticInput(t *testing.T)
|
||||
}
|
||||
return validators.Result{Decisions: decisions}, nil
|
||||
}}, validatormetadata.ExecutionClassLLMBacked)
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh teh cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, semantic}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate(0, 1, "cat", "dog"), candidate(1, 0, "teh", "the"), candidate(2, 1, "teh", "the"), candidate(3, 1, "cat", "cat")}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh teh cat"}}}, Validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, semantic}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate(0, 1, "cat", "dog"), candidate(1, 0, "teh", "the"), candidate(2, 1, "teh", "the"), candidate(3, 1, "cat", "cat")}}
|
||||
result, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -58,14 +58,14 @@ func TestFeedbackKeepsUnavailableAndZeroBudgetShortlists(t *testing.T) {
|
||||
for _, budget := range []int{0, 1} {
|
||||
t.Run(fmt.Sprint(budget), func(t *testing.T) {
|
||||
var later int
|
||||
first := fakeValidator{name: "first", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
first := fakeValidator{name: "first", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
var decisions []validators.Decision
|
||||
for _, p := range req.CandidateProposal {
|
||||
decisions = append(decisions, validators.Decision{ProposalIndex: p.ProposalIndex, Disposition: validators.DispositionUnavailable, ReasonCode: "unavailable"})
|
||||
}
|
||||
return validators.Result{Decisions: decisions}, nil
|
||||
}}
|
||||
second := fakeValidator{name: "second", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
second := fakeValidator{name: "second", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
later = len(req.CandidateProposal)
|
||||
var decisions []validators.Decision
|
||||
for _, p := range req.CandidateProposal {
|
||||
@@ -73,7 +73,7 @@ func TestFeedbackKeepsUnavailableAndZeroBudgetShortlists(t *testing.T) {
|
||||
}
|
||||
return validators.Result{Decisions: decisions}, nil
|
||||
}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{first, second}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{first, second}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}}
|
||||
result, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -95,8 +95,8 @@ func TestRepairFindingsUseSafeUnavailableEvidence(t *testing.T) {
|
||||
|
||||
func TestRevalidationArtifactsUseProposalAndAttemptCoordinates(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{
|
||||
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}},
|
||||
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}},
|
||||
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}},
|
||||
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}},
|
||||
}}
|
||||
validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
|
||||
if err != nil {
|
||||
@@ -108,7 +108,7 @@ func TestRevalidationArtifactsUseProposalAndAttemptCoordinates(t *testing.T) {
|
||||
var paths []string
|
||||
for _, index := range []int{2, 3} {
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar_1", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validator}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, ValidationLLMClient: client, Interactions: sink, SectionIndex: §ion, RepairProposalIndex: &index, RepairAttempt: &attempt}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar_1", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{validator}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, ValidationLLMClient: client, Interactions: sink, SectionIndex: §ion, RepairProposalIndex: &index, RepairAttempt: &attempt}
|
||||
result, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -149,11 +149,11 @@ func TestRevalidationArtifactsUseProposalAndAttemptCoordinates(t *testing.T) {
|
||||
|
||||
func TestFeedbackReportsProtectionAndBlankResult(t *testing.T) {
|
||||
identity := glossary.Identity{Collection: "party", ID: "jesters"}
|
||||
protected := fakeValidator{name: "protected_terms", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
protected := fakeValidator{name: "protected_terms", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
return validators.ValidateProtectedTerms(req)
|
||||
}}
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "Jesters", CorrectedText: ""}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "spoken_word", ProtectedTerms: []glossary.Term{{Spelling: "Jesters", Identities: []glossary.Identity{identity}}}, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Jesters"}}}, Validators: []contracts.Validator{protected, validators.NonEmptyResultingSegmentValidator{}}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "spoken_word", ProtectedTerms: []glossary.Term{{Spelling: "Jesters", Identities: []glossary.Identity{identity}}}, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Jesters"}}}, Validators: []validators.Validator{protected, validators.NonEmptyResultingSegmentValidator{}}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}}
|
||||
result, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -2,6 +2,7 @@ package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||
@@ -27,7 +28,7 @@ func (m orderedModule) Key() string { return m.spec.ModuleKey }
|
||||
func (m orderedModule) ReplacementPolicy() proposals.ReplacementPolicy {
|
||||
return proposals.ReplacementPolicyRequireUnique
|
||||
}
|
||||
func (m orderedModule) Validators() []contracts.Validator { return nil }
|
||||
func (m orderedModule) Validators() []frameworkvalidators.Validator { return nil }
|
||||
func (m orderedModule) Propose(_ context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
m.seen <- req
|
||||
original, corrected := "party", "Party"
|
||||
|
||||
@@ -24,6 +24,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
||||
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
|
||||
)
|
||||
|
||||
type repairClient struct {
|
||||
@@ -52,7 +53,7 @@ func TestRepairBranchesRetargetAndPreserveOtherApprovals(t *testing.T) {
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 4}},
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 0, OriginalText: "x", CorrectedText: "y"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 5}},
|
||||
}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}, {ID: 2, Text: "uh hello"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.NonEmptyResultingSegmentValidator{}}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}, {ID: 2, Text: "uh hello"}}}, Validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.NonEmptyResultingSegmentValidator{}}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{}}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -92,7 +93,7 @@ func TestRepairRequestCarriesLiteralPreview(t *testing.T) {
|
||||
Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}},
|
||||
Policy: tc.policy,
|
||||
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 81, Text: tc.segment}}},
|
||||
Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.NonEmptyResultingSegmentValidator{}, rejectUntilRepair()},
|
||||
Validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.NonEmptyResultingSegmentValidator{}, rejectUntilRepair()},
|
||||
SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate},
|
||||
Exchange: &structuredoutput.Exchange{},
|
||||
}
|
||||
@@ -131,7 +132,7 @@ func TestRepairRequestRefreshesPreviewAfterRetarget(t *testing.T) {
|
||||
`{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`,
|
||||
}}
|
||||
var validatedIDs []int
|
||||
reject := fakeValidator{name: "editorial_review", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
reject := fakeValidator{name: "editorial_review", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
validatedIDs = append(validatedIDs, req.CandidateProposal[0].TargetSegmentID)
|
||||
return validators.Result{Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Disposition: validators.DispositionRejected, ReasonCode: "llm_rejected", Message: "The context does not support this edit."}}}, nil
|
||||
}}
|
||||
@@ -139,9 +140,10 @@ func TestRepairRequestRefreshesPreviewAfterRetarget(t *testing.T) {
|
||||
Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 2}}},
|
||||
Policy: proposals.ReplacementPolicyRequireUnique,
|
||||
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 81, Text: "cat"}, {ID: 104, Text: "uh hello"}}},
|
||||
Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, reject},
|
||||
Validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validatormetadata.Wrap(reject, validatormetadata.ExecutionClassDeterministic)},
|
||||
SectionEnriched: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 81, OriginalText: "cat", CorrectedText: "dog"}}},
|
||||
Exchange: &structuredoutput.Exchange{},
|
||||
Interactions: llm.NewDiagnosticsWriter(t.TempDir()),
|
||||
}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
@@ -154,6 +156,15 @@ func TestRepairRequestRefreshesPreviewAfterRetarget(t *testing.T) {
|
||||
if len(validatedIDs) != 2 || validatedIDs[0] != 81 || validatedIDs[1] != 104 {
|
||||
t.Fatalf("validated segment IDs: %v", validatedIDs)
|
||||
}
|
||||
for _, decisions := range [][]ValidatorDecisionRecord{initial.decisions, branches[0].Attempts[0].Decisions} {
|
||||
if len(decisions) == 0 {
|
||||
t.Fatal("missing decision evidence")
|
||||
}
|
||||
record := decisions[len(decisions)-1]
|
||||
if record.ValidatorName != "editorial_review" || record.ExecutionClass != validatormetadata.ExecutionClassDeterministic {
|
||||
t.Fatalf("lost execution class override: %+v", record)
|
||||
}
|
||||
}
|
||||
for i, want := range []struct{ before, after string }{{"cat", "dog"}, {"uh hello", "hello"}} {
|
||||
var payload proposal_generation.RepairRequest
|
||||
if err := json.Unmarshal([]byte(client.calls[i].Variables["RepairRequestJSON"]), &payload); err != nil {
|
||||
@@ -183,7 +194,7 @@ func TestRepairBranchTerminalOutcomes(t *testing.T) {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
client := &repairClient{t: t, outputs: tc.outputs, err: tc.err}
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: tc.budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: tc.budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -211,7 +222,7 @@ func TestRunnerAppliesRevisionsInOriginalOrder(t *testing.T) {
|
||||
client := &repairClient{t: t, outputs: []string{`{"proposal_ref":2,"decision":{"action":"revise","correction":{"id":1,"original_text":"cat","corrected_text":"fox"}}}`,
|
||||
`{"proposal_ref":3,"decision":{"action":"revise","correction":{"id":1,"original_text":"bird","corrected_text":"owl"}}}`,
|
||||
}}
|
||||
module := repairModule{outcomeModule{fakeModule: fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, rejectUntilRepair(0)}}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
module := repairModule{outcomeModule{fakeModule: fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, rejectUntilRepair(0)}}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
return contracts.ProposalResult{Proposals: candidates, SegmentReferences: []int{1, 1, 1}, Exchange: &structuredoutput.Exchange{}}, nil
|
||||
}}}
|
||||
runner := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}})
|
||||
@@ -244,7 +255,7 @@ func TestRepairRevalidatesEarlierApprovalAndRetainsUnrelatedApprovalOnFailure(t
|
||||
}{{"revision", false}, {"provider failure", true}} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
calls := 0
|
||||
check := fakeValidator{name: "custom", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
check := fakeValidator{name: "custom", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
var decisions []validators.Decision
|
||||
for _, candidate := range req.CandidateProposal {
|
||||
calls++
|
||||
@@ -256,7 +267,7 @@ func TestRepairRevalidatesEarlierApprovalAndRetainsUnrelatedApprovalOnFailure(t
|
||||
if tc.failure {
|
||||
client.err = &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonProviderFailed, Message: "failed"}
|
||||
}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat bird"}}}, Validators: []contracts.Validator{check, rejectUntilRepair(0)}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat bird"}}}, Validators: []validators.Validator{check, rejectUntilRepair(0)}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{}}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -279,7 +290,7 @@ func TestRepairRevalidatesEarlierApprovalAndRetainsUnrelatedApprovalOnFailure(t
|
||||
func TestRepairInterruptionRetainsAttempt(t *testing.T) {
|
||||
client := &repairClient{t: t, err: &structuredoutput.Failure{Class: structuredoutput.FailureCanceled, Reason: outcomes.ReasonCanceled, Message: "canceled", Cause: context.Canceled}}
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -297,7 +308,7 @@ func TestRepairArtifactsAreUnique(t *testing.T) {
|
||||
{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "fox"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}},
|
||||
}
|
||||
section := 4
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar_1", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: candidates, SectionIndex: §ion, Exchange: &structuredoutput.Exchange{}, Interactions: llm.NewDiagnosticsWriter(t.TempDir())}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar_1", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{rejectUntilRepair()}, SectionEnriched: candidates, SectionIndex: §ion, Exchange: &structuredoutput.Exchange{}, Interactions: llm.NewDiagnosticsWriter(t.TempDir())}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -355,7 +366,7 @@ func TestRepairArtifactFailureDoesNotChangeOutcome(t *testing.T) {
|
||||
}
|
||||
sink := llm.NewDiagnosticsWriter(path)
|
||||
candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}, Interactions: sink}
|
||||
input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}, Interactions: sink}
|
||||
initial, err := validateSectionCandidates(context.Background(), input)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -382,8 +393,8 @@ func TestNoEffectAndUnavailableOnlyDoNotStartRepair(t *testing.T) {
|
||||
}
|
||||
|
||||
// rejectUntilRepair supplies a semantic rejection at the external validator seam.
|
||||
func rejectUntilRepair(preserved ...int) contracts.Validator {
|
||||
return fakeValidator{name: "editorial_review", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
func rejectUntilRepair(preserved ...int) validators.Validator {
|
||||
return fakeValidator{name: "editorial_review", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
var decisions []validators.Decision
|
||||
for _, p := range req.CandidateProposal {
|
||||
approved := req.RepairAttempt != nil
|
||||
|
||||
@@ -69,13 +69,14 @@ type WorkUnitOutcome struct {
|
||||
}
|
||||
|
||||
type ValidatorDecisionRecord struct {
|
||||
ValidatorName string `json:"validator_name"`
|
||||
ProposalIndex int `json:"proposal_index"`
|
||||
Approved bool `json:"approved"`
|
||||
Disposition validators.DecisionDisposition `json:"disposition"`
|
||||
ReasonCode string `json:"reason_code"`
|
||||
Message string `json:"message"`
|
||||
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
|
||||
ExecutionClass validatormetadata.ExecutionClass `json:"-"`
|
||||
ValidatorName string `json:"validator_name"`
|
||||
ProposalIndex int `json:"proposal_index"`
|
||||
Approved bool `json:"approved"`
|
||||
Disposition validators.DecisionDisposition `json:"disposition"`
|
||||
ReasonCode string `json:"reason_code"`
|
||||
Message string `json:"message"`
|
||||
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
|
||||
}
|
||||
|
||||
type ValidatorRejectedChange struct {
|
||||
@@ -650,7 +651,7 @@ type validateSectionCandidatesInput struct {
|
||||
Policy proposals.ReplacementPolicy
|
||||
WorkingTranscript *schema.Transcript
|
||||
SectionIndex *int
|
||||
Validators []contracts.Validator
|
||||
Validators []validators.Validator
|
||||
SectionEnriched []proposals.EnrichedCorrectionProposal
|
||||
ValidationLLMClient contracts.StructuredLLMClient
|
||||
Interactions structuredoutput.InteractionSink
|
||||
@@ -673,7 +674,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
||||
eligible := append([]proposals.EnrichedCorrectionProposal(nil), input.SectionEnriched...)
|
||||
feedback := input.Spec.Settings.LLM.CorrectionRepairAttempts > 0
|
||||
var blocked, noEffect map[int]bool
|
||||
evidence := contracts.ValidationRequest{WorkingTranscript: input.WorkingTranscript, Section: input.Section, ReplacementPolicy: input.Policy, IndependentEvidence: feedback}
|
||||
evidence := validators.Request{WorkingTranscript: input.WorkingTranscript, Section: input.Section, ReplacementPolicy: input.Policy, IndependentEvidence: feedback}
|
||||
if feedback {
|
||||
blocked, noEffect = make(map[int]bool), make(map[int]bool)
|
||||
evidence.Previews = make(map[int]proposals.SegmentPreviewResult)
|
||||
@@ -686,11 +687,12 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
||||
if err := ctx.Err(); err != nil {
|
||||
return validateSectionCandidatesResult{decisions: decisions, rejected: rejected, operations: operations, filtered: filtered}, err
|
||||
}
|
||||
class := validatormetadata.ClassOf(validator)
|
||||
submitted := eligible
|
||||
if feedback {
|
||||
submitted = feedbackCandidates(input, validator, noEffect, evidence)
|
||||
submitted = feedbackCandidates(input, validator.Name(), class == validatormetadata.ExecutionClassLLMBacked, noEffect, evidence)
|
||||
}
|
||||
vResult, err := validator.Validate(ctx, contracts.ValidationRequest{
|
||||
vResult, err := validator.Validate(ctx, validators.Request{
|
||||
StageIndex: &input.Spec.Index, StageID: input.Spec.ID,
|
||||
WorkingTranscript: input.WorkingTranscript,
|
||||
Section: evidence.Section, Previews: evidence.Previews,
|
||||
@@ -728,17 +730,20 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
||||
byIndex[p.ProposalIndex] = p
|
||||
}
|
||||
|
||||
for _, d := range vResult.Decisions {
|
||||
if feedback && validator.Name() == "original_text_presence" && d.Approved {
|
||||
preview := evidence.Preview(byIndex[d.ProposalIndex])
|
||||
if preview.SkipReason == proposals.SkipReasonAmbiguousOriginal {
|
||||
d.Approved = false
|
||||
d.Disposition = validators.DispositionRejected
|
||||
d.ReasonCode = string(proposals.SkipReasonAmbiguousOriginal)
|
||||
d.Message = "original_text matches multiple locations in the target segment"
|
||||
}
|
||||
dispositions := make([]validators.DecisionDisposition, len(vResult.Decisions))
|
||||
for i, decision := range vResult.Decisions {
|
||||
disposition, contractErr := decision.EffectiveDisposition()
|
||||
if contractErr != nil {
|
||||
return validateSectionCandidatesResult{
|
||||
decisions: decisions, rejected: rejected, operations: operations, filtered: filtered,
|
||||
}, fmt.Errorf("validator %q returned inconsistent decisions: %w", validator.Name(), errors.Join(err, contractErr))
|
||||
}
|
||||
if feedback && !d.Approved {
|
||||
dispositions[i] = disposition
|
||||
}
|
||||
for i, d := range vResult.Decisions {
|
||||
disposition := dispositions[i]
|
||||
approved := disposition == validators.DispositionApproved
|
||||
if feedback && !approved {
|
||||
blocked[d.ProposalIndex] = true
|
||||
if d.ReasonCode == validators.ReasonNoEffect {
|
||||
noEffect[d.ProposalIndex] = true
|
||||
@@ -746,18 +751,19 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
||||
}
|
||||
decisions = append(decisions, ValidatorDecisionRecord{
|
||||
ValidatorName: validator.Name(),
|
||||
ExecutionClass: class,
|
||||
ProposalIndex: d.ProposalIndex,
|
||||
Approved: d.Approved,
|
||||
Disposition: d.Disposition,
|
||||
Approved: approved,
|
||||
Disposition: disposition,
|
||||
ReasonCode: d.ReasonCode,
|
||||
Message: d.Message,
|
||||
DiagnosticArtifactPath: d.DiagnosticArtifactPath,
|
||||
})
|
||||
if d.Approved {
|
||||
if approved {
|
||||
nextEligible = append(nextEligible, byIndex[d.ProposalIndex])
|
||||
continue
|
||||
}
|
||||
if d.Disposition != validators.DispositionUnavailable {
|
||||
if disposition != validators.DispositionUnavailable {
|
||||
if reason, ok := validators.DefectiveProposalReason(d.ReasonCode); ok {
|
||||
filtered = append(filtered, outcomes.FilteredProposal{ProposalIndex: d.ProposalIndex, Reason: reason, Count: 1})
|
||||
}
|
||||
@@ -771,7 +777,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida
|
||||
TargetSegmentID: p.TargetSegmentID,
|
||||
OriginalText: p.OriginalText,
|
||||
CorrectedText: p.CorrectedText,
|
||||
Disposition: d.Disposition,
|
||||
Disposition: disposition,
|
||||
ReasonCode: d.ReasonCode,
|
||||
Message: d.Message,
|
||||
})
|
||||
@@ -882,11 +888,13 @@ func attributeApplyFiltering(sections []chunking.Section, approved []proposals.E
|
||||
}
|
||||
}
|
||||
|
||||
func reorderValidatorsForPipeline(in []contracts.Validator) ([]contracts.Validator, map[string]int) {
|
||||
deterministic := make([]contracts.Validator, 0, len(in))
|
||||
llmBacked := make([]contracts.Validator, 0, len(in))
|
||||
func reorderValidatorsForPipeline(in []validators.Validator) ([]validators.Validator, map[string]int) {
|
||||
deterministic := make([]validators.Validator, 0, len(in))
|
||||
llmBacked := make([]validators.Validator, 0, len(in))
|
||||
for _, validator := range in {
|
||||
if validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked {
|
||||
class := validatormetadata.ClassOf(validator)
|
||||
validator = validatormetadata.Wrap(validator, class)
|
||||
if class == validatormetadata.ExecutionClassLLMBacked {
|
||||
llmBacked = append(llmBacked, validator)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -43,7 +43,7 @@ func (f fakeFactory) ModuleForSpec(spec contracts.ModuleRunSpec) (contracts.Tran
|
||||
type fakeModule struct {
|
||||
key string
|
||||
policy proposals.ReplacementPolicy
|
||||
validators []contracts.Validator
|
||||
validators []validators.Validator
|
||||
proposeF func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error)
|
||||
}
|
||||
|
||||
@@ -52,6 +52,33 @@ type outcomeModule struct {
|
||||
proposeResult func(contracts.ProposalRequest) (contracts.ProposalResult, error)
|
||||
}
|
||||
|
||||
func TestRunnerStopsOnContradictoryValidatorDecision(t *testing.T) {
|
||||
candidate := proposals.EnrichedCorrectionProposal{
|
||||
CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"},
|
||||
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 7},
|
||||
}
|
||||
contractFailure := errors.New("validator also returned a runtime error")
|
||||
for _, cause := range []error{nil, contractFailure} {
|
||||
contradictory := fakeValidator{name: "invalid", validateF: func(validators.Request) (validators.Result, error) {
|
||||
return validators.Result{Decisions: []validators.Decision{{
|
||||
ProposalIndex: 7, Approved: true, Disposition: validators.DispositionUnavailable,
|
||||
}}}, cause
|
||||
}}
|
||||
later := fakeValidator{name: "later", validateF: func(validators.Request) (validators.Result, error) {
|
||||
t.Fatal("candidate advanced past a contradictory decision")
|
||||
return validators.Result{}, nil
|
||||
}}
|
||||
result, err := validateSectionCandidates(context.Background(), validateSectionCandidatesInput{
|
||||
Validators: []validators.Validator{validators.ProposalShapeValidator{}, contradictory, later},
|
||||
SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate},
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "contradictory") || (cause != nil && !errors.Is(err, cause)) ||
|
||||
len(result.approved) != 0 || len(result.decisions) != 1 || result.decisions[0].ValidatorName != "proposal_shape" {
|
||||
t.Fatalf("contradictory result = %+v (%v)", result, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRecoverableProposalFailureAllowsLaterModule(t *testing.T) {
|
||||
failed := outcomeModule{fakeModule: fakeModule{key: "first", policy: proposals.ReplacementPolicyRequireUnique}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
return contracts.ProposalResult{Operations: []outcomes.Operation{{Reference: outcomes.Reference{ModuleKey: "first", ModuleInstance: "first", Kind: outcomes.OperationProposal}, FailureReason: outcomes.ReasonTransportFailed}}}, nil
|
||||
@@ -74,7 +101,7 @@ func TestRunnerRecoverableProposalFailureAllowsLaterModule(t *testing.T) {
|
||||
|
||||
func TestRunnerOnlyFullyValidatedCandidatesApplyAndRecoveriesMerge(t *testing.T) {
|
||||
secondSaw := make([]int, 0)
|
||||
partial := fakeValidator{name: "partial", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
partial := fakeValidator{name: "partial", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
if len(req.CandidateProposal) != 2 {
|
||||
t.Fatalf("expected two candidates after shape filtering, got %d", len(req.CandidateProposal))
|
||||
}
|
||||
@@ -86,13 +113,13 @@ func TestRunnerOnlyFullyValidatedCandidatesApplyAndRecoveriesMerge(t *testing.T)
|
||||
{Reference: outcomes.Reference{Kind: outcomes.OperationValidation}, FailureReason: outcomes.ReasonProviderFailed},
|
||||
}}, nil
|
||||
}}
|
||||
last := fakeValidator{name: "last", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
last := fakeValidator{name: "last", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
for _, candidate := range req.CandidateProposal {
|
||||
secondSaw = append(secondSaw, candidate.ProposalIndex)
|
||||
}
|
||||
return validators.Result{ValidatorName: "last", Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: true, Disposition: validators.DispositionApproved}}}, nil
|
||||
}}
|
||||
module := outcomeModule{fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{validators.ProposalShapeValidator{}, partial, last}}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
module := outcomeModule{fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{validators.ProposalShapeValidator{}, partial, last}}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
return contracts.ProposalResult{Proposals: []proposals.CorrectionProposal{
|
||||
{TargetSegmentID: 0, OriginalText: "bad", CorrectedText: "x"},
|
||||
{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"},
|
||||
@@ -120,10 +147,10 @@ func TestRunnerOnlyFullyValidatedCandidatesApplyAndRecoveriesMerge(t *testing.T)
|
||||
func TestFatalValidatorNeverReturnsUncheckedCandidates(t *testing.T) {
|
||||
origin := errors.New("validator setup failed")
|
||||
proposal := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 7}}
|
||||
broken := fakeValidator{name: "broken", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
broken := fakeValidator{name: "broken", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
return validators.Result{Operations: []outcomes.Operation{{Reference: outcomes.Reference{Kind: outcomes.OperationValidation}, FailureReason: outcomes.ReasonSetupFailure}}}, origin
|
||||
}}
|
||||
result, err := validateSectionCandidates(context.Background(), validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "m", InstanceName: "m"}, Validators: []contracts.Validator{broken}, SectionEnriched: []proposals.EnrichedCorrectionProposal{proposal}})
|
||||
result, err := validateSectionCandidates(context.Background(), validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "m", InstanceName: "m"}, Validators: []validators.Validator{broken}, SectionEnriched: []proposals.EnrichedCorrectionProposal{proposal}})
|
||||
if !errors.Is(err, origin) || len(result.approved) != 0 || len(result.operations) != 1 {
|
||||
t.Fatalf("fatal result must retain facts without approving candidates: result=%+v err=%v", result, err)
|
||||
}
|
||||
@@ -131,7 +158,7 @@ func TestFatalValidatorNeverReturnsUncheckedCandidates(t *testing.T) {
|
||||
|
||||
func TestCandidateCaptureIsExplicitAndIncludesFilteredProposals(t *testing.T) {
|
||||
module := fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique,
|
||||
validators: []contracts.Validator{validators.ProposalShapeValidator{}},
|
||||
validators: []validators.Validator{validators.ProposalShapeValidator{}},
|
||||
proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 0, OriginalText: "bad", CorrectedText: "good"}}, nil
|
||||
}}
|
||||
@@ -160,7 +187,7 @@ func (m outcomeModule) Propose(_ context.Context, req contracts.ProposalRequest)
|
||||
|
||||
func (m fakeModule) Key() string { return m.key }
|
||||
func (m fakeModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy }
|
||||
func (m fakeModule) Validators() []contracts.Validator { return m.validators }
|
||||
func (m fakeModule) Validators() []validators.Validator { return m.validators }
|
||||
func (m fakeModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
if m.proposeF == nil {
|
||||
return contracts.ProposalResult{}, nil
|
||||
@@ -171,11 +198,11 @@ func (m fakeModule) Propose(ctx context.Context, req contracts.ProposalRequest)
|
||||
|
||||
type fakeValidator struct {
|
||||
name string
|
||||
validateF func(req contracts.ValidationRequest) (validators.Result, error)
|
||||
validateF func(req validators.Request) (validators.Result, error)
|
||||
}
|
||||
|
||||
func (v fakeValidator) Name() string { return v.name }
|
||||
func (v fakeValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (validators.Result, error) {
|
||||
func (v fakeValidator) Validate(ctx context.Context, req validators.Request) (validators.Result, error) {
|
||||
_ = ctx
|
||||
return v.validateF(req)
|
||||
}
|
||||
@@ -191,19 +218,19 @@ func (v classifiedFakeValidator) ExecutionClass() validatormetadata.ExecutionCla
|
||||
|
||||
func TestReorderValidatorsDeterministicBeforeLLMBacked(t *testing.T) {
|
||||
llm := classifiedFakeValidator{
|
||||
fakeValidator: fakeValidator{name: "llm", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
fakeValidator: fakeValidator{name: "llm", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
return validators.Result{ValidatorName: "llm"}, nil
|
||||
}},
|
||||
class: validatormetadata.ExecutionClassLLMBacked,
|
||||
}
|
||||
deterministic := classifiedFakeValidator{
|
||||
fakeValidator: fakeValidator{name: "deterministic", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
fakeValidator: fakeValidator{name: "deterministic", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
return validators.Result{ValidatorName: "deterministic"}, nil
|
||||
}},
|
||||
class: validatormetadata.ExecutionClassDeterministic,
|
||||
}
|
||||
|
||||
ordered, _ := reorderValidatorsForPipeline([]contracts.Validator{llm, deterministic})
|
||||
ordered, _ := reorderValidatorsForPipeline([]validators.Validator{llm, deterministic})
|
||||
if len(ordered) != 2 {
|
||||
t.Fatalf("expected 2 validators, got %d", len(ordered))
|
||||
}
|
||||
@@ -213,17 +240,17 @@ func TestReorderValidatorsDeterministicBeforeLLMBacked(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestReorderValidatorsDefaultsUnclassifiedToDeterministic(t *testing.T) {
|
||||
unclassified := fakeValidator{name: "plain", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
unclassified := fakeValidator{name: "plain", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
return validators.Result{ValidatorName: "plain"}, nil
|
||||
}}
|
||||
llm := classifiedFakeValidator{
|
||||
fakeValidator: fakeValidator{name: "llm", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
fakeValidator: fakeValidator{name: "llm", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
return validators.Result{ValidatorName: "llm"}, nil
|
||||
}},
|
||||
class: validatormetadata.ExecutionClassLLMBacked,
|
||||
}
|
||||
|
||||
ordered, _ := reorderValidatorsForPipeline([]contracts.Validator{llm, unclassified})
|
||||
ordered, _ := reorderValidatorsForPipeline([]validators.Validator{llm, unclassified})
|
||||
if len(ordered) != 2 {
|
||||
t.Fatalf("expected 2 validators, got %d", len(ordered))
|
||||
}
|
||||
@@ -553,7 +580,7 @@ func TestRunnerValidationStartsBeforeAllSectionProposalsComplete(t *testing.T) {
|
||||
|
||||
validator := fakeValidator{
|
||||
name: "capture",
|
||||
validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
validateF: func(req validators.Request) (validators.Result, error) {
|
||||
if len(req.CandidateProposal) == 1 && req.CandidateProposal[0].TargetSegmentID == 1 {
|
||||
select {
|
||||
case validatorStarted <- struct{}{}:
|
||||
@@ -577,7 +604,7 @@ func TestRunnerValidationStartsBeforeAllSectionProposalsComplete(t *testing.T) {
|
||||
"m": fakeModule{
|
||||
key: "m",
|
||||
policy: proposals.ReplacementPolicyRequireUnique,
|
||||
validators: []contracts.Validator{validator},
|
||||
validators: []validators.Validator{validator},
|
||||
proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
seg := req.WorkingTranscript.Segments[0]
|
||||
if req.Section != nil && req.Section.Index == 1 {
|
||||
@@ -630,7 +657,7 @@ func (c *captureContractStructuredClient) CompleteStructured(ctx context.Context
|
||||
if target, ok := out.(*validators.LLMValidationResponse); ok {
|
||||
*target = validators.LLMValidationResponse{
|
||||
Validations: []validators.LLMValidationDecision{
|
||||
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
|
||||
{CorrectionIndex: "1", Approved: true, Reason: "ok"},
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -662,7 +689,7 @@ func (c *stageAwareStructuredClient) CompleteStructured(ctx context.Context, req
|
||||
target := out.(*validators.LLMValidationResponse)
|
||||
*target = validators.LLMValidationResponse{
|
||||
Validations: []validators.LLMValidationDecision{
|
||||
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
|
||||
{CorrectionIndex: "1", Approved: true, Reason: "ok"},
|
||||
},
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{}, nil
|
||||
@@ -734,7 +761,7 @@ func TestRunnerRepeatedModuleInstanceNames(t *testing.T) {
|
||||
|
||||
func TestRunnerValidatorApprovedProposalApplied(t *testing.T) {
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}}
|
||||
allowAll := fakeValidator{name: "allow", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
allowAll := fakeValidator{name: "allow", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
decisions := make([]validators.Decision, len(req.CandidateProposal))
|
||||
for i, p := range req.CandidateProposal {
|
||||
decisions[i] = validators.Decision{ProposalIndex: p.ProposalIndex, Approved: true, ReasonCode: validators.ReasonApproved, Message: "approved"}
|
||||
@@ -745,7 +772,7 @@ func TestRunnerValidatorApprovedProposalApplied(t *testing.T) {
|
||||
"m": fakeModule{
|
||||
key: "m",
|
||||
policy: proposals.ReplacementPolicyRequireUnique,
|
||||
validators: []contracts.Validator{allowAll},
|
||||
validators: []validators.Validator{allowAll},
|
||||
proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil
|
||||
},
|
||||
@@ -766,7 +793,7 @@ func TestRunnerValidatorApprovedProposalApplied(t *testing.T) {
|
||||
|
||||
func TestRunnerValidatorRejectedProposalNotApplied(t *testing.T) {
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}}
|
||||
rejectAll := fakeValidator{name: "reject", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
rejectAll := fakeValidator{name: "reject", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
decisions := make([]validators.Decision, len(req.CandidateProposal))
|
||||
for i, p := range req.CandidateProposal {
|
||||
decisions[i] = validators.Decision{ProposalIndex: p.ProposalIndex, Approved: false, ReasonCode: validators.ReasonNoEffect, Message: "rejected"}
|
||||
@@ -774,7 +801,7 @@ func TestRunnerValidatorRejectedProposalNotApplied(t *testing.T) {
|
||||
return validators.Result{ValidatorName: "reject", Decisions: decisions}, nil
|
||||
}}
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{rejectAll}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{rejectAll}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil
|
||||
}},
|
||||
}})
|
||||
@@ -792,7 +819,7 @@ func TestRunnerValidatorRejectedProposalNotApplied(t *testing.T) {
|
||||
|
||||
func TestRunnerMultipleValidatorsRunInOrderAndFilterSurvivors(t *testing.T) {
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "one two"}}}
|
||||
first := fakeValidator{name: "first", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
first := fakeValidator{name: "first", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
if len(req.CandidateProposal) != 2 {
|
||||
t.Fatalf("expected first validator to see 2 candidates, got %d", len(req.CandidateProposal))
|
||||
}
|
||||
@@ -804,14 +831,14 @@ func TestRunnerMultipleValidatorsRunInOrderAndFilterSurvivors(t *testing.T) {
|
||||
},
|
||||
}, nil
|
||||
}}
|
||||
second := fakeValidator{name: "second", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
second := fakeValidator{name: "second", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
if len(req.CandidateProposal) != 1 {
|
||||
t.Fatalf("expected second validator to see only survivors, got %d", len(req.CandidateProposal))
|
||||
}
|
||||
return validators.Result{ValidatorName: "second", Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: true, ReasonCode: validators.ReasonApproved, Message: "ok"}}}, nil
|
||||
}}
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{first, second}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{first, second}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{
|
||||
{TargetSegmentID: 1, OriginalText: "one", CorrectedText: "ONE"},
|
||||
{TargetSegmentID: 1, OriginalText: "two", CorrectedText: "TWO"},
|
||||
@@ -833,11 +860,11 @@ func TestRunnerValidatorCardinalityErrorStopsPipelineWithPartialProgress(t *test
|
||||
good := fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil
|
||||
}}
|
||||
badValidator := fakeValidator{name: "bad", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
badValidator := fakeValidator{name: "bad", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
// Missing one decision triggers cardinality error.
|
||||
return validators.Result{ValidatorName: "bad", Decisions: nil}, nil
|
||||
}}
|
||||
bad := fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{badValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
bad := fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{badValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}}, nil
|
||||
}}
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"m1": good, "m2": bad}})
|
||||
@@ -856,11 +883,11 @@ func TestRunnerValidatorCardinalityErrorStopsPipelineWithPartialProgress(t *test
|
||||
|
||||
func TestRunnerApplicationSkipAfterValidatorApprovalReported(t *testing.T) {
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "word word"}}}
|
||||
allow := fakeValidator{name: "allow", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
allow := fakeValidator{name: "allow", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
return validators.Result{ValidatorName: "allow", Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: true, ReasonCode: validators.ReasonApproved, Message: "ok"}}}, nil
|
||||
}}
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{allow}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{allow}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "word", CorrectedText: "term"}}, nil
|
||||
}},
|
||||
}})
|
||||
@@ -880,7 +907,7 @@ func TestRunnerApplyConflictAfterTwoInitiallyValidDeletions(t *testing.T) {
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "a b"}}}
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": outcomeModule{
|
||||
fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NonEmptyResultingSegmentValidator{}}},
|
||||
fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NonEmptyResultingSegmentValidator{}}},
|
||||
proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
return contracts.ProposalResult{
|
||||
Proposals: []proposals.CorrectionProposal{
|
||||
@@ -945,14 +972,14 @@ func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contr
|
||||
}
|
||||
|
||||
func TestRunnerLLMValidatorApprovalApplied(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}}}
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}}}
|
||||
llmValidator, err := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
|
||||
if err != nil {
|
||||
t.Fatalf("NewLLMBackedValidator: %v", err)
|
||||
}
|
||||
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil
|
||||
}},
|
||||
}})
|
||||
@@ -971,10 +998,10 @@ func TestRunnerLLMValidatorApprovalApplied(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunnerLLMValidatorRejectionPreventsApplication(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: false, Reason: "reject"}}}}}
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: false, Reason: "reject"}}}}}
|
||||
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil
|
||||
}},
|
||||
}})
|
||||
@@ -996,13 +1023,13 @@ func TestRunnerLLMValidatorRejectionPreventsApplication(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Reason: "bad index"}}}}}
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "99", Approved: true, Reason: "bad index"}}}}}
|
||||
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m1": fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil
|
||||
}},
|
||||
"m2": fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
"m2": fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}}, nil
|
||||
}},
|
||||
}})
|
||||
@@ -1028,13 +1055,13 @@ func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress(
|
||||
|
||||
func TestRunnerReconciledSurvivorsCompleteLaterValidation(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{
|
||||
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
|
||||
{CorrectionIndex: 3, Approved: true, Reason: "ok"},
|
||||
{CorrectionIndex: 99, Approved: true, Reason: "unexpected"},
|
||||
{CorrectionIndex: "1", Approved: true, Reason: "ok"},
|
||||
{CorrectionIndex: "3", Approved: true, Reason: "ok"},
|
||||
{CorrectionIndex: "99", Approved: true, Reason: "unexpected"},
|
||||
}}}}
|
||||
first, _ := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview)
|
||||
var laterIndices []int
|
||||
later := fakeValidator{name: "later_review", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
|
||||
later := fakeValidator{name: "later_review", validateF: func(req validators.Request) (validators.Result, error) {
|
||||
decisions := make([]validators.Decision, 0, len(req.CandidateProposal))
|
||||
for _, candidate := range req.CandidateProposal {
|
||||
laterIndices = append(laterIndices, candidate.ProposalIndex)
|
||||
@@ -1047,7 +1074,7 @@ func TestRunnerReconciledSurvivorsCompleteLaterValidation(t *testing.T) {
|
||||
}
|
||||
return validators.Result{ValidatorName: "later_review", Decisions: decisions}, nil
|
||||
}}
|
||||
module := outcomeModule{fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{first, validatormetadata.Wrap(later, validatormetadata.ExecutionClassLLMBacked)}}, proposeResult: func(contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
module := outcomeModule{fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{first, validatormetadata.Wrap(later, validatormetadata.ExecutionClassLLMBacked)}}, proposeResult: func(contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
return contracts.ProposalResult{Proposals: []proposals.CorrectionProposal{
|
||||
{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"},
|
||||
{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"},
|
||||
@@ -1077,7 +1104,7 @@ func TestRunnerLLMValidatorMissingDecisionRejectsBatch(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{}}}}
|
||||
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil
|
||||
}},
|
||||
}})
|
||||
@@ -1094,12 +1121,12 @@ func TestRunnerLLMValidatorMissingDecisionRejectsBatch(t *testing.T) {
|
||||
|
||||
func TestRunnerLLMValidatorDuplicateDecisionRejectsBatch(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{
|
||||
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
|
||||
{CorrectionIndex: 1, Approved: false, Reason: "dup"},
|
||||
{CorrectionIndex: "1", Approved: true, Reason: "ok"},
|
||||
{CorrectionIndex: "1", Approved: false, Reason: "dup"},
|
||||
}}}}
|
||||
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil
|
||||
}},
|
||||
}})
|
||||
@@ -1120,8 +1147,8 @@ func (lenEstimator) EstimateTokens(text string) int { return len(text) }
|
||||
|
||||
func TestRunnerLLMValidatorBatching(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{
|
||||
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}},
|
||||
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}},
|
||||
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}},
|
||||
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}},
|
||||
}}
|
||||
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
|
||||
llmValidator.SetTokenEstimator(lenEstimator{})
|
||||
@@ -1129,7 +1156,7 @@ func TestRunnerLLMValidatorBatching(t *testing.T) {
|
||||
cfg := runnerTestConfig()
|
||||
cfg.Validation.MaxPromptTokens = 260
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyReplaceAll, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyReplaceAll, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{
|
||||
{TargetSegmentID: 1, OriginalText: "alpha", CorrectedText: strings.Repeat("B", 40)},
|
||||
{TargetSegmentID: 1, OriginalText: "gamma", CorrectedText: strings.Repeat("D", 40)},
|
||||
@@ -1154,12 +1181,12 @@ func TestRunnerLLMValidatorBatching(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunnerLLMValidatorDiagnosticsWritten(t *testing.T) {
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "supported by context"}}}}}
|
||||
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "supported by context"}}}}}
|
||||
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
|
||||
cfg := runnerTestConfig()
|
||||
diagDir := t.TempDir()
|
||||
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
|
||||
return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil
|
||||
}},
|
||||
}})
|
||||
@@ -1208,7 +1235,7 @@ type proposalGenerationModule struct {
|
||||
|
||||
func (m proposalGenerationModule) Key() string { return m.key }
|
||||
func (m proposalGenerationModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy }
|
||||
func (m proposalGenerationModule) Validators() []contracts.Validator { return nil }
|
||||
func (m proposalGenerationModule) Validators() []validators.Validator { return nil }
|
||||
func (m proposalGenerationModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
result, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
|
||||
ModuleKey: req.RunSpec.ModuleKey,
|
||||
@@ -1294,10 +1321,10 @@ func TestFatalPartialDecisionsRejectUnknownAndDuplicateIdentity(t *testing.T) {
|
||||
{{ProposalIndex: 8, Approved: true}},
|
||||
{{ProposalIndex: 7, Approved: true}, {ProposalIndex: 7, Approved: true}},
|
||||
} {
|
||||
v := fakeValidator{name: "broken", validateF: func(contracts.ValidationRequest) (validators.Result, error) {
|
||||
v := fakeValidator{name: "broken", validateF: func(validators.Request) (validators.Result, error) {
|
||||
return validators.Result{Decisions: decisions}, origin
|
||||
}}
|
||||
result, err := validateSectionCandidates(context.Background(), validateSectionCandidatesInput{Validators: []contracts.Validator{v}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}})
|
||||
result, err := validateSectionCandidates(context.Background(), validateSectionCandidatesInput{Validators: []validators.Validator{v}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}})
|
||||
if !errors.Is(err, origin) || len(result.decisions) != 0 || len(result.approved) != 0 {
|
||||
t.Fatalf("invalid partial evidence accepted or origin lost: %+v %v", result, err)
|
||||
}
|
||||
|
||||
@@ -37,6 +37,7 @@ func (v OriginalTextPresenceValidator) Name() string { return "original_text_pre
|
||||
|
||||
func (v OriginalTextPresenceValidator) Validate(_ context.Context, req Request) (Result, error) {
|
||||
section := req.sectionLookup()
|
||||
req.Section = section
|
||||
|
||||
decisions := make([]Decision, 0, len(req.CandidateProposal))
|
||||
for _, c := range req.CandidateProposal {
|
||||
@@ -49,6 +50,10 @@ func (v OriginalTextPresenceValidator) Validate(_ context.Context, req Request)
|
||||
decisions = append(decisions, rejection(c.ProposalIndex, ReasonMissingOriginalText, "original_text was not found in current segment text"))
|
||||
continue
|
||||
}
|
||||
if req.IndependentEvidence && req.Preview(c).SkipReason == proposals.SkipReasonAmbiguousOriginal {
|
||||
decisions = append(decisions, rejection(c.ProposalIndex, string(proposals.SkipReasonAmbiguousOriginal), "original_text matches multiple locations in the target segment"))
|
||||
continue
|
||||
}
|
||||
decisions = append(decisions, approval(c.ProposalIndex))
|
||||
}
|
||||
if err := EnforceDecisionCardinality(req.CandidateProposal, decisions); err != nil {
|
||||
|
||||
@@ -25,15 +25,15 @@ func TestValidationContextCarriesAdjacentEvidenceAndRelevantDefinitions(t *testi
|
||||
{ID: "lyra", Name: "Lyra", Description: "The host"},
|
||||
{ID: "unrelated", Name: "Unrelated", Description: "Not mentioned"},
|
||||
}
|
||||
payload, immediate := BuildLLMValidationRequest("spoken_form_plausibility", LLMValidatorTypeSpokenFormPlausibility, req)
|
||||
if len(immediate) != 0 || len(payload.Items) != 1 {
|
||||
payload, immediate := buildValidationItems(req)
|
||||
if len(immediate) != 0 || len(payload) != 1 {
|
||||
t.Fatalf("payload: %+v immediate: %+v", payload, immediate)
|
||||
}
|
||||
item := payload.Items[0]
|
||||
item := payload[0]
|
||||
if len(item.NeighboringSegments) != 2 || item.NeighboringSegments[0].Position != "preceding" || item.NeighboringSegments[1].Position != "following" || len(item.GlossaryEntries) != 2 || item.GlossaryEntries[0].Name != "Jesters" || item.GlossaryEntries[1].Name != "Lyra" {
|
||||
t.Fatalf("context: %+v", item)
|
||||
}
|
||||
variables, err := BuildValidationVariables(payload.Items, "")
|
||||
variables, err := BuildValidationVariables(payload, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -41,7 +41,7 @@ func TestValidationContextCarriesAdjacentEvidenceAndRelevantDefinitions(t *testi
|
||||
if err := json.Unmarshal([]byte(variables["PayloadJSON"]), &rendered); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(variables["PayloadJSON"], `"segment_id":`) || rendered[0].CorrectionIndex != 1 || payload.Items[0].CorrectionIndex != 7 {
|
||||
if strings.Contains(variables["PayloadJSON"], `"segment_id":`) || rendered[0].CorrectionIndex != 1 || payload[0].CorrectionIndex != 7 {
|
||||
t.Fatalf("leaked internal identity or mutated request: %s", variables["PayloadJSON"])
|
||||
}
|
||||
if rendered[0].Speaker != "Guest" || len(rendered[0].NeighboringSegments) != 2 || rendered[0].NeighboringSegments[0].Speaker != "Host" || rendered[0].GlossaryEntries[0].Description != "A traveling troupe" {
|
||||
@@ -51,15 +51,15 @@ func TestValidationContextCarriesAdjacentEvidenceAndRelevantDefinitions(t *testi
|
||||
req.WorkingTranscript.Segments = req.WorkingTranscript.Segments[1:2]
|
||||
req.WorkingTranscript.Segments[0].Speaker = ""
|
||||
req.Glossary = nil
|
||||
isolated, _ := BuildLLMValidationRequest("editorial_review", LLMValidatorTypeEditorialReview, req)
|
||||
if isolated.Items[0].Speaker != "" || len(isolated.Items[0].NeighboringSegments) != 0 || len(isolated.Items[0].GlossaryEntries) != 0 {
|
||||
isolated, _ := buildValidationItems(req)
|
||||
if isolated[0].Speaker != "" || len(isolated[0].NeighboringSegments) != 0 || len(isolated[0].GlossaryEntries) != 0 {
|
||||
t.Fatalf("unexpected external context: %+v", isolated)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationContextFitsBudgetWithoutDiscardingCandidate(t *testing.T) {
|
||||
estimator := chunking.NewSimpleTokenEstimator()
|
||||
base := LLMValidationItem{CorrectionIndex: 7, SegmentID: 1, OriginalText: "teh", CorrectedText: "the", OriginalSegmentText: "teh cat", CorrectedSegmentText: "the cat"}
|
||||
base := LLMValidationItem{CorrectionIndex: 7, OriginalText: "teh", CorrectedText: "the", OriginalSegmentText: "teh cat", CorrectedSegmentText: "the cat"}
|
||||
required, err := estimateBatchTokens(estimator, []LLMValidationItem{base})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -89,12 +89,12 @@ func TestValidationContextSelectsGlossaryForEachReplacement(t *testing.T) {
|
||||
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(7, "gestures", "Jesters"), mk(11, "gestures", "Actors")})
|
||||
req.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Those gestures are back."}}}
|
||||
req.Glossary = []glossary.Entry{{ID: "actors", Name: "Actors"}, {ID: "jesters", Name: "Jesters"}}
|
||||
payload, immediate := BuildLLMValidationRequest("editorial_review", LLMValidatorTypeEditorialReview, req)
|
||||
if len(immediate) != 0 || len(payload.Items) != 2 {
|
||||
payload, immediate := buildValidationItems(req)
|
||||
if len(immediate) != 0 || len(payload) != 2 {
|
||||
t.Fatalf("payload: %+v immediate: %+v", payload, immediate)
|
||||
}
|
||||
for i, name := range []string{"Jesters", "Actors"} {
|
||||
entries := payload.Items[i].GlossaryEntries
|
||||
entries := payload[i].GlossaryEntries
|
||||
if len(entries) != 1 || entries[0].Name != name {
|
||||
t.Fatalf("candidate %d context: %+v", i, entries)
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||
ck "go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
@@ -50,28 +51,7 @@ func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) {
|
||||
}
|
||||
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
|
||||
})
|
||||
dir := t.TempDir()
|
||||
main := filepath.Join(dir, "config.yml")
|
||||
if err := os.WriteFile(main, []byte("version: 3\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte("profiles:\n - {id: base, endpoint: 'https://example.invalid/v1', model: local-model, api_key_required: false}\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
loaded, err := config.Load(ck.Input{Source: ck.File(main)}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runtime, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
settings := loaded.Config.LLM
|
||||
settings.TransientRetries, settings.RepairAttempts = 0, 1
|
||||
client, err := runtime.NewRoleClient("validation", "base", settings, llm.CallHooks{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client := testValidationClient(t, transport)
|
||||
validator, err := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -91,3 +71,98 @@ func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The transport supplies provider responses; PromptKit performs structured
|
||||
// validation and Audita decodes the same response through the role client.
|
||||
func testValidationClient(t *testing.T, transport http.RoundTripper) structuredoutput.Client {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
main := filepath.Join(dir, "config.yml")
|
||||
if err := os.WriteFile(main, []byte("version: 3\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte("profiles:\n - {id: base, endpoint: 'https://example.invalid/v1', model: local-model, api_key_required: false}\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
loaded, err := config.Load(ck.Input{Source: ck.File(main)}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runtime, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
settings := loaded.Config.LLM
|
||||
settings.TransientRetries, settings.RepairAttempts = 0, 1
|
||||
client, err := runtime.NewRoleClient("validation", "base", settings, llm.CallHooks{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return client
|
||||
}
|
||||
|
||||
func TestValidationDecodesExactReferencesBeforeReconciliation(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, response string
|
||||
indices []int
|
||||
want []DecisionDisposition
|
||||
reasons []string
|
||||
failure outcomes.ReasonCode
|
||||
}{
|
||||
{
|
||||
name: "usable decision beside extra",
|
||||
response: `{"validations":[{"correction_index":1.00e0,"approved":true,"reason":"accept"},{"correction_index":9223372036854775808,"approved":true,"reason":"extra"}]}`,
|
||||
indices: []int{7},
|
||||
want: []DecisionDisposition{DispositionApproved},
|
||||
reasons: []string{ReasonApproved},
|
||||
},
|
||||
{
|
||||
name: "usable siblings beside duplicate and omission",
|
||||
response: `{"validations":[{"correction_index":1e0,"approved":true,"reason":"accept"},{"correction_index":2,"approved":true,"reason":"duplicate"},{"correction_index":20e-1,"approved":false,"reason":"duplicate"},{"correction_index":4.000,"approved":false,"reason":"reject"},{"correction_index":9223372036854775808,"approved":true,"reason":"extra"}]}`,
|
||||
indices: []int{7, 11, 14, 23},
|
||||
want: []DecisionDisposition{DispositionApproved, DispositionUnavailable, DispositionUnavailable, DispositionRejected},
|
||||
reasons: []string{ReasonApproved, ReasonDuplicateValidationDecision, ReasonMissingValidationDecision, "llm_rejected"},
|
||||
failure: outcomes.ReasonValidationDecisionsInvalid,
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
calls := 0
|
||||
client := testValidationClient(t, validationTransport(func(req *http.Request) (*http.Response, error) {
|
||||
calls++
|
||||
content, _ := json.Marshal(tc.response)
|
||||
body := fmt.Sprintf(`{"choices":[{"message":{"content":%s}}]}`, content)
|
||||
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: req}, nil
|
||||
}))
|
||||
var candidates []proposals.EnrichedCorrectionProposal
|
||||
for _, index := range tc.indices {
|
||||
candidates = append(candidates, mk(index, "gestures", "Jesters"))
|
||||
}
|
||||
validator, err := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sink := &captureValidationDiagnosticsWriter{}
|
||||
req := makeReq(candidates)
|
||||
req.LLMClient = client
|
||||
req.Interactions = sink
|
||||
result, err := validator.Validate(context.Background(), req)
|
||||
if err != nil || calls != 1 || len(result.Decisions) != len(tc.want) || len(result.Operations) != 1 {
|
||||
t.Fatalf("result=%+v calls=%d error=%v", result, calls, err)
|
||||
}
|
||||
for i, decision := range result.Decisions {
|
||||
if decision.ProposalIndex != tc.indices[i] || decision.Disposition != tc.want[i] || decision.ReasonCode != tc.reasons[i] {
|
||||
t.Fatalf("decision %d: %+v", i, decision)
|
||||
}
|
||||
}
|
||||
operation := result.Operations[0]
|
||||
if operation.FailureReason != tc.failure || (tc.failure == "" && (len(operation.RecoveryReasons) != 1 || operation.RecoveryReasons[0] != outcomes.RecoveryValidationDecisionsFiltered)) {
|
||||
t.Fatalf("operation: %+v", operation)
|
||||
}
|
||||
payload := sink.lastResponsePayload.(map[string]any)
|
||||
diagnostics := payload["reconciliation"].(validationReconciliation)
|
||||
if len(diagnostics.IgnoredIndices) != 1 || diagnostics.IgnoredIndices[0] != "9223372036854775808" || payload["final_content"] != tc.response {
|
||||
t.Fatalf("reference evidence changed: %+v", payload)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package validators
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||
)
|
||||
@@ -21,7 +23,6 @@ type StructuredLLMClient = structuredoutput.Client
|
||||
// BuildValidationVariables substitutes a one-based reference for each batch.
|
||||
type LLMValidationItem struct {
|
||||
CorrectionIndex int `json:"correction_index"`
|
||||
SegmentID int `json:"-"`
|
||||
Speaker string `json:"speaker"`
|
||||
OriginalText string `json:"original_text"`
|
||||
CorrectedText string `json:"corrected_text"`
|
||||
@@ -32,21 +33,11 @@ type LLMValidationItem struct {
|
||||
GlossaryEntries []glossary.Entry `json:"glossary_entries,omitempty"`
|
||||
}
|
||||
|
||||
// LLMValidationRequest is the canonical request model for LLM-backed validators.
|
||||
type LLMValidationRequest struct {
|
||||
ValidatorName string `json:"validator_name"`
|
||||
ValidatorType LLMValidatorType `json:"validator_type"`
|
||||
ModuleKey string `json:"module_key"`
|
||||
ModuleInstance string `json:"module_instance"`
|
||||
ReplacementPolicy string `json:"replacement_policy"`
|
||||
Items []LLMValidationItem `json:"corrections"`
|
||||
}
|
||||
|
||||
// LLMValidationDecision contains a request-local correction_index, not a proposal index.
|
||||
type LLMValidationDecision struct {
|
||||
CorrectionIndex int `json:"correction_index"`
|
||||
Approved bool `json:"approved"`
|
||||
Reason string `json:"reason"`
|
||||
CorrectionIndex json.Number `json:"correction_index"`
|
||||
Approved bool `json:"approved"`
|
||||
Reason string `json:"reason"`
|
||||
}
|
||||
|
||||
type LLMValidationResponse struct {
|
||||
|
||||
@@ -48,9 +48,9 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
|
||||
}
|
||||
failed := &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonTransportFailed, Message: "model transport failed"}
|
||||
client := &scriptedValidationClient{replies: []validationReply{
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}},
|
||||
{completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 1}}, err: failed},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}},
|
||||
}}
|
||||
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
|
||||
validator.SetTokenEstimator(perItemEstimator{})
|
||||
@@ -73,8 +73,8 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
|
||||
|
||||
survivors := []proposals.EnrichedCorrectionProposal{candidates[0], candidates[2]}
|
||||
nextClient := &scriptedValidationClient{replies: []validationReply{
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}},
|
||||
}}
|
||||
nextValidator, _ := NewLLMBackedValidator("meaning_reversal_review", LLMValidatorTypeMeaningReversal)
|
||||
nextValidator.SetTokenEstimator(perItemEstimator{})
|
||||
@@ -101,7 +101,7 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
|
||||
|
||||
func TestValidationDomainInvalidAfterRepairIsAnError(t *testing.T) {
|
||||
client := &scriptedValidationClient{replies: []validationReply{{
|
||||
response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Reason: "ok"}}},
|
||||
response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "99", Approved: true, Reason: "ok"}}},
|
||||
completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 2, StructuralRepairs: 1}},
|
||||
}}}
|
||||
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
|
||||
@@ -116,8 +116,8 @@ func TestValidationDomainInvalidAfterRepairIsAnError(t *testing.T) {
|
||||
func TestValidationExtraDecisionsMergeActualRecoveries(t *testing.T) {
|
||||
client := &scriptedValidationClient{replies: []validationReply{{
|
||||
response: LLMValidationResponse{Validations: []LLMValidationDecision{
|
||||
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
|
||||
{CorrectionIndex: 99, Approved: true, Reason: "extra"},
|
||||
{CorrectionIndex: "1", Approved: true, Reason: "ok"},
|
||||
{CorrectionIndex: "99", Approved: true, Reason: "extra"},
|
||||
}},
|
||||
completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 3, TransientRetries: 1, StructuralRepairs: 1}},
|
||||
}}}
|
||||
@@ -137,9 +137,9 @@ func TestValidationExtraDecisionsMergeActualRecoveries(t *testing.T) {
|
||||
func TestValidationFatalFailureKeepsEarlierFactsAndStops(t *testing.T) {
|
||||
cause := errors.New("broken embedded schema")
|
||||
client := &scriptedValidationClient{replies: []validationReply{
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}},
|
||||
{err: &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "model setup failed", Cause: cause}},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}},
|
||||
{response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}},
|
||||
}}
|
||||
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
|
||||
validator.SetTokenEstimator(perItemEstimator{})
|
||||
|
||||
@@ -2,6 +2,7 @@ package validators
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
@@ -9,6 +10,7 @@ import (
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/jsonnumber"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/stagename"
|
||||
@@ -58,8 +60,8 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
if req.LLMClient == nil {
|
||||
return Result{}, fmt.Errorf("LLM-backed validator %q requires a structured LLM client", v.name)
|
||||
}
|
||||
validationReq, immediate := BuildLLMValidationRequest(v.name, v.validatorType, req)
|
||||
if len(validationReq.Items) == 0 {
|
||||
items, immediate := buildValidationItems(req)
|
||||
if len(items) == 0 {
|
||||
all := append([]Decision(nil), immediate...)
|
||||
sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex })
|
||||
if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil {
|
||||
@@ -73,7 +75,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
|
||||
maxTokens = req.Settings.Validation.MaxPromptTokens
|
||||
}
|
||||
|
||||
batches, oversized, err := prepareValidationBatches(validationReq.Items, maxTokens, v.estimator)
|
||||
batches, oversized, err := prepareValidationBatches(items, maxTokens, v.estimator)
|
||||
operations := make([]outcomes.Operation, 0)
|
||||
llmDecisions := make([]Decision, 0)
|
||||
partialResult := func() Result {
|
||||
@@ -239,7 +241,7 @@ func errPayload(err error) any {
|
||||
return map[string]any{"error": err.Error()}
|
||||
}
|
||||
|
||||
func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorType, req Request) (LLMValidationRequest, []Decision) {
|
||||
func buildValidationItems(req Request) ([]LLMValidationItem, []Decision) {
|
||||
items := make([]LLMValidationItem, 0, len(req.CandidateProposal))
|
||||
immediate := make([]Decision, 0)
|
||||
req.Section = req.sectionLookup()
|
||||
@@ -268,7 +270,6 @@ func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorT
|
||||
neighbors, glossaryEntries := glossary.validationContext(segments, position, preview.CorrectedSegmentText)
|
||||
items = append(items, LLMValidationItem{
|
||||
CorrectionIndex: p.ProposalIndex,
|
||||
SegmentID: p.TargetSegmentID,
|
||||
Speaker: seg.Speaker,
|
||||
OriginalText: p.OriginalText,
|
||||
CorrectedText: p.CorrectedText,
|
||||
@@ -280,31 +281,25 @@ func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorT
|
||||
})
|
||||
}
|
||||
|
||||
return LLMValidationRequest{
|
||||
ValidatorName: validatorName,
|
||||
ValidatorType: validatorType,
|
||||
ModuleKey: req.ModuleKey,
|
||||
ModuleInstance: req.ModuleInstance,
|
||||
ReplacementPolicy: string(req.ReplacementPolicy),
|
||||
Items: items,
|
||||
}, immediate
|
||||
return items, immediate
|
||||
}
|
||||
|
||||
type validationReconciliation struct {
|
||||
Decisions []Decision `json:"decisions"`
|
||||
UnavailableIndices []int `json:"unavailable_indices,omitempty"`
|
||||
IgnoredIndices []int `json:"ignored_indices,omitempty"`
|
||||
Decisions []Decision `json:"decisions"`
|
||||
UnavailableIndices []int `json:"unavailable_indices,omitempty"`
|
||||
IgnoredIndices []json.Number `json:"ignored_indices,omitempty"`
|
||||
}
|
||||
|
||||
func reconcileLLMResponse(items []LLMValidationItem, response LLMValidationResponse) validationReconciliation {
|
||||
result := validationReconciliation{Decisions: make([]Decision, 0, len(items))}
|
||||
seen := make(map[int][]LLMValidationDecision, len(items))
|
||||
for _, d := range response.Validations {
|
||||
if d.CorrectionIndex < 1 || d.CorrectionIndex > len(items) {
|
||||
index, ok := jsonnumber.PositiveIndex(d.CorrectionIndex.String(), len(items))
|
||||
if !ok {
|
||||
result.IgnoredIndices = append(result.IgnoredIndices, d.CorrectionIndex)
|
||||
continue
|
||||
}
|
||||
seen[d.CorrectionIndex] = append(seen[d.CorrectionIndex], d)
|
||||
seen[index] = append(seen[index], d)
|
||||
}
|
||||
|
||||
for i, item := range items {
|
||||
|
||||
@@ -4,6 +4,8 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -74,7 +76,7 @@ func mk(index int, orig, corr string) proposals.EnrichedCorrectionProposal {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildLLMValidationRequestPreservesLiteralDeletionAndSpacing(t *testing.T) {
|
||||
func TestValidationPayloadPreservesLiteralDeletionAndSpacing(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, segment, original, replacement, result string
|
||||
}{
|
||||
@@ -84,27 +86,25 @@ func TestBuildLLMValidationRequestPreservesLiteralDeletionAndSpacing(t *testing.
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(7, tc.original, tc.replacement)})
|
||||
req.WorkingTranscript.Segments[0].Text = tc.segment
|
||||
payload, immediate := BuildLLMValidationRequest("editorial_review", LLMValidatorTypeEditorialReview, req)
|
||||
if len(immediate) != 0 || len(payload.Items) != 1 {
|
||||
payload, immediate := buildValidationItems(req)
|
||||
if len(immediate) != 0 || len(payload) != 1 {
|
||||
t.Fatalf("expected previewable request: %+v, %+v", payload, immediate)
|
||||
}
|
||||
raw, err := json.Marshal(payload)
|
||||
variables, err := BuildValidationVariables(payload, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var decoded struct {
|
||||
ValidatorName string `json:"validator_name"`
|
||||
Corrections []map[string]any `json:"corrections"`
|
||||
}
|
||||
raw := []byte(variables["PayloadJSON"])
|
||||
var decoded []map[string]any
|
||||
if err := json.Unmarshal(raw, &decoded); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if decoded.ValidatorName != "editorial_review" || len(decoded.Corrections) != 1 {
|
||||
t.Fatalf("wrong validator identity or item count: %s", raw)
|
||||
if len(decoded) != 1 {
|
||||
t.Fatalf("wrong item count: %s", raw)
|
||||
}
|
||||
item := decoded.Corrections[0]
|
||||
item := decoded[0]
|
||||
value, present := item["corrected_text"]
|
||||
if !present || value != tc.replacement || item["original_text"] != tc.original || item["original_segment_text"] != tc.segment || item["corrected_segment_text"] != tc.result || item["correction_index"] != float64(7) {
|
||||
if !present || value != tc.replacement || item["original_text"] != tc.original || item["original_segment_text"] != tc.segment || item["corrected_segment_text"] != tc.result || item["correction_index"] != float64(1) {
|
||||
t.Fatalf("literal spans or full segment text lost: %s", raw)
|
||||
}
|
||||
})
|
||||
@@ -148,8 +148,8 @@ func TestChunkLLMValidationItemsMultipleBatchesStableNoDropNoDup(t *testing.T) {
|
||||
|
||||
func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) {
|
||||
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
|
||||
{CorrectionIndex: 1, Approved: true, Reason: ""},
|
||||
{CorrectionIndex: 2, Approved: false, Reason: "bad"},
|
||||
{CorrectionIndex: "1", Approved: true, Reason: ""},
|
||||
{CorrectionIndex: "2", Approved: false, Reason: "bad"},
|
||||
}}}}
|
||||
v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
|
||||
if err != nil {
|
||||
@@ -203,10 +203,10 @@ func TestLLMBackedValidatorProviderMalformedEnvelopePropagates(t *testing.T) {
|
||||
|
||||
func TestLLMBackedValidatorReconcilesDecisionIdentities(t *testing.T) {
|
||||
approve := func(index int) LLMValidationDecision {
|
||||
return LLMValidationDecision{CorrectionIndex: index, Approved: true, Reason: "accepted"}
|
||||
return LLMValidationDecision{CorrectionIndex: json.Number(strconv.Itoa(index)), Approved: true, Reason: "accepted"}
|
||||
}
|
||||
reject := func(index int) LLMValidationDecision {
|
||||
return LLMValidationDecision{CorrectionIndex: index, Reason: "editorial concern"}
|
||||
return LLMValidationDecision{CorrectionIndex: json.Number(strconv.Itoa(index)), Reason: "editorial concern"}
|
||||
}
|
||||
type expectedDecision struct {
|
||||
index int
|
||||
@@ -218,13 +218,13 @@ func TestLLMBackedValidatorReconcilesDecisionIdentities(t *testing.T) {
|
||||
entries []LLMValidationDecision
|
||||
want []expectedDecision
|
||||
unavailable []int
|
||||
ignored []int
|
||||
ignored []json.Number
|
||||
}{
|
||||
{"internal index is not a local reference", []LLMValidationDecision{approve(7), approve(2), reject(3)}, []expectedDecision{{7, false, ReasonMissingValidationDecision}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, []int{7}, []int{7}},
|
||||
{"missing and extra", []LLMValidationDecision{approve(1), reject(3), approve(99)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonMissingValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, []int{99}},
|
||||
{"internal index is not a local reference", []LLMValidationDecision{approve(7), approve(2), reject(3)}, []expectedDecision{{7, false, ReasonMissingValidationDecision}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, []int{7}, []json.Number{"7"}},
|
||||
{"missing and extra", []LLMValidationDecision{approve(1), reject(3), approve(99)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonMissingValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, []json.Number{"99"}},
|
||||
{"conflicting duplicate", []LLMValidationDecision{approve(1), approve(2), reject(2), reject(3)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonDuplicateValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
|
||||
{"identical duplicate", []LLMValidationDecision{approve(1), approve(2), approve(2), reject(3)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonDuplicateValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil},
|
||||
{"extra only", []LLMValidationDecision{approve(99), reject(3), approve(1), approve(99), approve(2)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, []int{99, 99}},
|
||||
{"extra only", []LLMValidationDecision{approve(99), reject(3), approve(1), approve(99), approve(2)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, []json.Number{"99", "99"}},
|
||||
{"empty", nil, []expectedDecision{{7, false, ReasonMissingValidationDecision}, {11, false, ReasonMissingValidationDecision}, {18, false, ReasonMissingValidationDecision}}, []int{7, 11, 18}, nil},
|
||||
{"reordered", []LLMValidationDecision{reject(3), approve(2), approve(1)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, nil},
|
||||
} {
|
||||
@@ -264,7 +264,7 @@ func TestLLMBackedValidatorReconcilesDecisionIdentities(t *testing.T) {
|
||||
t.Fatalf("supplied model decisions lost from diagnostic: %+v", decoded)
|
||||
}
|
||||
reconciliation := payload["reconciliation"].(validationReconciliation)
|
||||
if !equalIndices(reconciliation.UnavailableIndices, tc.unavailable) || !equalIndices(reconciliation.IgnoredIndices, tc.ignored) || len(reconciliation.Decisions) != len(tc.want) {
|
||||
if !equalIndices(reconciliation.UnavailableIndices, tc.unavailable) || !slices.Equal(reconciliation.IgnoredIndices, tc.ignored) || len(reconciliation.Decisions) != len(tc.want) {
|
||||
t.Fatalf("diagnostic reconciliation=%+v", reconciliation)
|
||||
}
|
||||
})
|
||||
@@ -285,7 +285,7 @@ func equalIndices(a, b []int) bool {
|
||||
|
||||
func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *testing.T) {
|
||||
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
|
||||
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
|
||||
{CorrectionIndex: "1", Approved: true, Reason: "ok"},
|
||||
}}}}
|
||||
v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
|
||||
huge := strings.Repeat("gestures ", 200)
|
||||
@@ -331,7 +331,7 @@ func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *tes
|
||||
|
||||
func TestLLMBackedValidatorDiagnosticsIncludeSchemaMetadata(t *testing.T) {
|
||||
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
|
||||
{CorrectionIndex: 1, Approved: true, Reason: "ok"},
|
||||
{CorrectionIndex: "1", Approved: true, Reason: "ok"},
|
||||
}}}}
|
||||
writer := &captureValidationDiagnosticsWriter{}
|
||||
v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
|
||||
@@ -370,7 +370,7 @@ func TestEveryValidationTypeSelectsEmbeddedPromptAndVariables(t *testing.T) {
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(string(tc.kind), func(t *testing.T) {
|
||||
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}}}
|
||||
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}}}
|
||||
validator, err := NewLLMBackedValidator(string(tc.kind), tc.kind)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -395,7 +395,7 @@ func TestEveryValidationTypeSelectsEmbeddedPromptAndVariables(t *testing.T) {
|
||||
func TestValidationDiagnosticFailureDoesNotChangeDecision(t *testing.T) {
|
||||
cause := errors.New("diagnostic storage unavailable")
|
||||
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
|
||||
req.LLMClient = &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}}}
|
||||
req.LLMClient = &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}}}
|
||||
sink := &captureValidationDiagnosticsWriter{err: cause}
|
||||
req.Interactions = sink
|
||||
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
|
||||
|
||||
@@ -62,6 +62,25 @@ type Decision struct {
|
||||
DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"`
|
||||
}
|
||||
|
||||
// EffectiveDisposition resolves an omitted disposition from the declared
|
||||
// approval and detects inconsistent explicit decisions.
|
||||
func (d Decision) EffectiveDisposition() (DecisionDisposition, error) {
|
||||
switch d.Disposition {
|
||||
case "":
|
||||
if d.Approved {
|
||||
return DispositionApproved, nil
|
||||
}
|
||||
return DispositionRejected, nil
|
||||
case DispositionApproved, DispositionRejected, DispositionUnavailable:
|
||||
if d.Approved == (d.Disposition == DispositionApproved) {
|
||||
return d.Disposition, nil
|
||||
}
|
||||
return "", fmt.Errorf("contradictory approval and disposition %q for proposal %d", d.Disposition, d.ProposalIndex)
|
||||
default:
|
||||
return "", fmt.Errorf("invalid decision disposition %q for proposal %d", d.Disposition, d.ProposalIndex)
|
||||
}
|
||||
}
|
||||
|
||||
// DecisionDisposition separates normal editorial rejection from a candidate
|
||||
// excluded because its model operation could not provide a trustworthy decision.
|
||||
type DecisionDisposition string
|
||||
@@ -80,7 +99,10 @@ type Result struct {
|
||||
Operations []outcomes.Operation `json:"operations,omitempty"`
|
||||
}
|
||||
|
||||
// Validator is the deterministic runtime validator interface.
|
||||
// Validator evaluates one section's candidates. An instance may be called
|
||||
// concurrently across sections and may keep synchronized state. Request data
|
||||
// is borrowed for the call; inputs must be copied before modification or
|
||||
// retention. Request.Preview may populate the caller's pass-local cache.
|
||||
type Validator interface {
|
||||
Name() string
|
||||
Validate(ctx context.Context, req Request) (Result, error)
|
||||
|
||||
@@ -54,3 +54,21 @@ func (r Request) Preview(candidate proposals.EnrichedCorrectionProposal) proposa
|
||||
}
|
||||
return preview
|
||||
}
|
||||
|
||||
// AcceptsIndependentCandidate decides which evidence can be submitted when
|
||||
// repair needs findings from each applicable validator.
|
||||
func (r Request) AcceptsIndependentCandidate(validatorName string, modelDriven bool, candidate proposals.EnrichedCorrectionProposal) bool {
|
||||
hasSpan := candidate.OriginalText != ""
|
||||
switch validatorName {
|
||||
case "proposal_shape":
|
||||
return true
|
||||
case "no_effect":
|
||||
return hasSpan
|
||||
case "original_text_presence":
|
||||
return hasSpan && candidate.TargetSegmentID > 0
|
||||
case "protected_terms", "non_empty_resulting_segment":
|
||||
return r.Preview(candidate).HasResult()
|
||||
default:
|
||||
return !modelDriven || r.Preview(candidate).Applicable
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package glossary
|
||||
|
||||
import (
|
||||
"context"
|
||||
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
|
||||
@@ -11,7 +12,7 @@ import (
|
||||
)
|
||||
|
||||
type Module struct {
|
||||
validators []contracts.Validator
|
||||
validators []frameworkvalidators.Validator
|
||||
}
|
||||
|
||||
func New() (*Module, error) {
|
||||
@@ -33,8 +34,8 @@ func (m *Module) ReplacementPolicy() proposals.ReplacementPolicy {
|
||||
return proposals.ReplacementPolicyReplaceAll
|
||||
}
|
||||
|
||||
func (m *Module) Validators() []contracts.Validator {
|
||||
return append([]contracts.Validator(nil), m.validators...)
|
||||
func (m *Module) Validators() []frameworkvalidators.Validator {
|
||||
return append([]frameworkvalidators.Validator(nil), m.validators...)
|
||||
}
|
||||
|
||||
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
|
||||
@@ -2,6 +2,7 @@ package grammar
|
||||
|
||||
import (
|
||||
"context"
|
||||
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
|
||||
@@ -11,7 +12,7 @@ import (
|
||||
)
|
||||
|
||||
type Module struct {
|
||||
validators []contracts.Validator
|
||||
validators []frameworkvalidators.Validator
|
||||
}
|
||||
|
||||
func New() (*Module, error) {
|
||||
@@ -33,8 +34,8 @@ func (m *Module) ReplacementPolicy() proposals.ReplacementPolicy {
|
||||
return proposals.ReplacementPolicyRequireUnique
|
||||
}
|
||||
|
||||
func (m *Module) Validators() []contracts.Validator {
|
||||
return append([]contracts.Validator(nil), m.validators...)
|
||||
func (m *Module) Validators() []frameworkvalidators.Validator {
|
||||
return append([]frameworkvalidators.Validator(nil), m.validators...)
|
||||
}
|
||||
|
||||
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
|
||||
@@ -2,6 +2,7 @@ package homophones
|
||||
|
||||
import (
|
||||
"context"
|
||||
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
|
||||
@@ -11,7 +12,7 @@ import (
|
||||
)
|
||||
|
||||
type Module struct {
|
||||
validators []contracts.Validator
|
||||
validators []frameworkvalidators.Validator
|
||||
}
|
||||
|
||||
func New() (*Module, error) {
|
||||
@@ -33,8 +34,8 @@ func (m *Module) ReplacementPolicy() proposals.ReplacementPolicy {
|
||||
return proposals.ReplacementPolicyRequireUnique
|
||||
}
|
||||
|
||||
func (m *Module) Validators() []contracts.Validator {
|
||||
return append([]contracts.Validator(nil), m.validators...)
|
||||
func (m *Module) Validators() []frameworkvalidators.Validator {
|
||||
return append([]frameworkvalidators.Validator(nil), m.validators...)
|
||||
}
|
||||
|
||||
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
|
||||
@@ -154,7 +154,7 @@ func TestHomophonesProtectedGlossaryTermBehaviorRejectsUnsafeCorrection(t *testi
|
||||
if err != nil {
|
||||
t.Fatalf("New error: %v", err)
|
||||
}
|
||||
var protectedValidator contracts.Validator
|
||||
var protectedValidator validators.Validator
|
||||
for _, v := range module.Validators() {
|
||||
if v.Name() == "protected_terms" {
|
||||
protectedValidator = v
|
||||
|
||||
@@ -2,6 +2,7 @@ package spoken_word
|
||||
|
||||
import (
|
||||
"context"
|
||||
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
|
||||
@@ -11,7 +12,7 @@ import (
|
||||
)
|
||||
|
||||
type Module struct {
|
||||
validators []contracts.Validator
|
||||
validators []frameworkvalidators.Validator
|
||||
}
|
||||
|
||||
func New() (*Module, error) {
|
||||
@@ -33,8 +34,8 @@ func (m *Module) ReplacementPolicy() proposals.ReplacementPolicy {
|
||||
return proposals.ReplacementPolicyRequireUnique
|
||||
}
|
||||
|
||||
func (m *Module) Validators() []contracts.Validator {
|
||||
return append([]contracts.Validator(nil), m.validators...)
|
||||
func (m *Module) Validators() []frameworkvalidators.Validator {
|
||||
return append([]frameworkvalidators.Validator(nil), m.validators...)
|
||||
}
|
||||
|
||||
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) {
|
||||
|
||||
@@ -154,7 +154,7 @@ func TestSpokenWordProtectedGlossaryTermBehaviorRejectsUnsafeCorrection(t *testi
|
||||
if err != nil {
|
||||
t.Fatalf("New error: %v", err)
|
||||
}
|
||||
var protectedValidator contracts.Validator
|
||||
var protectedValidator validators.Validator
|
||||
for _, v := range module.Validators() {
|
||||
if v.Name() == "protected_terms" {
|
||||
protectedValidator = v
|
||||
|
||||
@@ -2,10 +2,10 @@ package validators
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||
)
|
||||
|
||||
var builtInChains = map[string][]string{
|
||||
@@ -58,7 +58,7 @@ func BuiltInChainKeys(moduleKey string) ([]string, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func ResolveBuiltInChain(moduleKey string, registry *Registry) ([]contracts.Validator, error) {
|
||||
func ResolveBuiltInChain(moduleKey string, registry *Registry) ([]frameworkvalidators.Validator, error) {
|
||||
moduleKey = strings.TrimSpace(moduleKey)
|
||||
keys, err := BuiltInChainKeys(moduleKey)
|
||||
if err != nil {
|
||||
@@ -68,7 +68,7 @@ func ResolveBuiltInChain(moduleKey string, registry *Registry) ([]contracts.Vali
|
||||
registry = NewBuiltInRegistry()
|
||||
}
|
||||
|
||||
out := make([]contracts.Validator, 0, len(keys))
|
||||
out := make([]frameworkvalidators.Validator, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
v, buildErr := registry.MustBuild(key)
|
||||
if buildErr != nil {
|
||||
|
||||
@@ -1,14 +1,13 @@
|
||||
package editorial_review
|
||||
|
||||
import (
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
||||
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
|
||||
)
|
||||
|
||||
const Key = "editorial_review"
|
||||
|
||||
func New() (contracts.Validator, error) {
|
||||
func New() (frameworkvalidators.Validator, error) {
|
||||
v, err := frameworkvalidators.NewLLMBackedValidator(Key, frameworkvalidators.LLMValidatorTypeEditorialReview)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -1,14 +1,13 @@
|
||||
package meaning_reversal_review
|
||||
|
||||
import (
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
||||
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
|
||||
)
|
||||
|
||||
const Key = "meaning_reversal_review"
|
||||
|
||||
func New() (contracts.Validator, error) {
|
||||
func New() (frameworkvalidators.Validator, error) {
|
||||
v, err := frameworkvalidators.NewLLMBackedValidator(Key, frameworkvalidators.LLMValidatorTypeMeaningReversal)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -3,7 +3,7 @@ package metadata
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
||||
)
|
||||
|
||||
type ExecutionClass string
|
||||
@@ -37,11 +37,11 @@ var executionClassByKey = map[string]ExecutionClass{
|
||||
}
|
||||
|
||||
type ClassifiedValidator interface {
|
||||
contracts.Validator
|
||||
frameworkvalidators.Validator
|
||||
ExecutionClass() ExecutionClass
|
||||
}
|
||||
|
||||
func ClassOf(v contracts.Validator) ExecutionClass {
|
||||
func ClassOf(v frameworkvalidators.Validator) ExecutionClass {
|
||||
if v == nil {
|
||||
return ExecutionClassDeterministic
|
||||
}
|
||||
@@ -69,7 +69,7 @@ func ClassForKey(key string) ExecutionClass {
|
||||
return class
|
||||
}
|
||||
|
||||
func Wrap(v contracts.Validator, class ExecutionClass) contracts.Validator {
|
||||
func Wrap(v frameworkvalidators.Validator, class ExecutionClass) frameworkvalidators.Validator {
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
@@ -83,7 +83,7 @@ func Wrap(v contracts.Validator, class ExecutionClass) contracts.Validator {
|
||||
}
|
||||
|
||||
type wrappedValidator struct {
|
||||
contracts.Validator
|
||||
frameworkvalidators.Validator
|
||||
class ExecutionClass
|
||||
}
|
||||
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
||||
)
|
||||
|
||||
@@ -12,7 +11,7 @@ type unclassifiedValidator struct{}
|
||||
|
||||
func (u unclassifiedValidator) Name() string { return "unclassified" }
|
||||
|
||||
func (u unclassifiedValidator) Validate(_ context.Context, _ contracts.ValidationRequest) (frameworkvalidators.Result, error) {
|
||||
func (u unclassifiedValidator) Validate(_ context.Context, _ frameworkvalidators.Request) (frameworkvalidators.Result, error) {
|
||||
return frameworkvalidators.Result{ValidatorName: u.Name(), Decisions: nil}, nil
|
||||
}
|
||||
|
||||
@@ -22,7 +21,7 @@ type namedUnclassifiedValidator struct {
|
||||
|
||||
func (n namedUnclassifiedValidator) Name() string { return n.name }
|
||||
|
||||
func (n namedUnclassifiedValidator) Validate(_ context.Context, _ contracts.ValidationRequest) (frameworkvalidators.Result, error) {
|
||||
func (n namedUnclassifiedValidator) Validate(_ context.Context, _ frameworkvalidators.Request) (frameworkvalidators.Result, error) {
|
||||
return frameworkvalidators.Result{ValidatorName: n.Name(), Decisions: nil}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@ package validators_test
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/modules/glossary"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/modules/grammar"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/modules/homophones"
|
||||
@@ -20,30 +20,30 @@ func TestProductionModulesUseRegisteredValidatorKeys(t *testing.T) {
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
build func() ([]contracts.Validator, error)
|
||||
build func() ([]frameworkvalidators.Validator, error)
|
||||
}{
|
||||
{name: "glossary", build: func() ([]contracts.Validator, error) {
|
||||
{name: "glossary", build: func() ([]frameworkvalidators.Validator, error) {
|
||||
m, err := glossary.New()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return m.Validators(), nil
|
||||
}},
|
||||
{name: "homophones", build: func() ([]contracts.Validator, error) {
|
||||
{name: "homophones", build: func() ([]frameworkvalidators.Validator, error) {
|
||||
m, err := homophones.New()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return m.Validators(), nil
|
||||
}},
|
||||
{name: "spoken_word", build: func() ([]contracts.Validator, error) {
|
||||
{name: "spoken_word", build: func() ([]frameworkvalidators.Validator, error) {
|
||||
m, err := spoken_word.New()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return m.Validators(), nil
|
||||
}},
|
||||
{name: "grammar", build: func() ([]contracts.Validator, error) {
|
||||
{name: "grammar", build: func() ([]frameworkvalidators.Validator, error) {
|
||||
m, err := grammar.New()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -1,13 +1,12 @@
|
||||
package no_effect
|
||||
|
||||
import (
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
||||
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
|
||||
)
|
||||
|
||||
const Key = "no_effect"
|
||||
|
||||
func New() (contracts.Validator, error) {
|
||||
func New() (frameworkvalidators.Validator, error) {
|
||||
return validatormetadata.Wrap(frameworkvalidators.NoEffectValidator{}, validatormetadata.ExecutionClassDeterministic), nil
|
||||
}
|
||||
|
||||
@@ -1,13 +1,12 @@
|
||||
package non_empty_resulting_segment
|
||||
|
||||
import (
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
||||
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
|
||||
)
|
||||
|
||||
const Key = "non_empty_resulting_segment"
|
||||
|
||||
func New() (contracts.Validator, error) {
|
||||
func New() (frameworkvalidators.Validator, error) {
|
||||
return validatormetadata.Wrap(frameworkvalidators.NonEmptyResultingSegmentValidator{}, validatormetadata.ExecutionClassDeterministic), nil
|
||||
}
|
||||
|
||||
@@ -1,13 +1,12 @@
|
||||
package original_text_presence
|
||||
|
||||
import (
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
||||
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
|
||||
)
|
||||
|
||||
const Key = "original_text_presence"
|
||||
|
||||
func New() (contracts.Validator, error) {
|
||||
func New() (frameworkvalidators.Validator, error) {
|
||||
return validatormetadata.Wrap(frameworkvalidators.OriginalTextPresenceValidator{}, validatormetadata.ExecutionClassDeterministic), nil
|
||||
}
|
||||
|
||||
@@ -1,13 +1,12 @@
|
||||
package proposal_shape
|
||||
|
||||
import (
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
||||
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
|
||||
)
|
||||
|
||||
const Key = "proposal_shape"
|
||||
|
||||
func New() (contracts.Validator, error) {
|
||||
func New() (frameworkvalidators.Validator, error) {
|
||||
return validatormetadata.Wrap(frameworkvalidators.ProposalShapeValidator{}, validatormetadata.ExecutionClassDeterministic), nil
|
||||
}
|
||||
|
||||
@@ -3,14 +3,13 @@ package protected_terms
|
||||
import (
|
||||
"context"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
||||
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
|
||||
)
|
||||
|
||||
const Key = "protected_terms"
|
||||
|
||||
func New() (contracts.Validator, error) {
|
||||
func New() (frameworkvalidators.Validator, error) {
|
||||
return validatormetadata.Wrap(validator{}, validatormetadata.ExecutionClassDeterministic), nil
|
||||
}
|
||||
|
||||
@@ -18,6 +17,6 @@ type validator struct{}
|
||||
|
||||
func (v validator) Name() string { return Key }
|
||||
|
||||
func (v validator) Validate(_ context.Context, req contracts.ValidationRequest) (frameworkvalidators.Result, error) {
|
||||
func (v validator) Validate(_ context.Context, req frameworkvalidators.Request) (frameworkvalidators.Result, error) {
|
||||
return frameworkvalidators.ValidateProtectedTerms(req)
|
||||
}
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
||||
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
|
||||
@@ -50,7 +49,7 @@ func TestNewUsesGeneralBehaviorForGlossaryModule(t *testing.T) {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
|
||||
req := contracts.ValidationRequest{
|
||||
req := frameworkvalidators.Request{
|
||||
ModuleKey: "glossary",
|
||||
Glossary: tinyGlossary(),
|
||||
CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
||||
@@ -78,7 +77,7 @@ func TestNewRejectsProtectedTermChangeOutsideGlossaryStage(t *testing.T) {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
|
||||
req := contracts.ValidationRequest{
|
||||
req := frameworkvalidators.Request{
|
||||
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Jesters"}}},
|
||||
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
||||
ModuleKey: "homophones",
|
||||
@@ -115,7 +114,7 @@ func TestNewGlossaryStageRejectsProtectedTermChange(t *testing.T) {
|
||||
t.Fatalf("expected deterministic class, got %q", got)
|
||||
}
|
||||
|
||||
req := contracts.ValidationRequest{
|
||||
req := frameworkvalidators.Request{
|
||||
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Jesters"}}},
|
||||
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
||||
ModuleKey: "glossary",
|
||||
@@ -146,7 +145,7 @@ func TestNewGlossaryStageAllowsProtectedTermInsertion(t *testing.T) {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
|
||||
req := contracts.ValidationRequest{
|
||||
req := frameworkvalidators.Request{
|
||||
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "gestures"}}},
|
||||
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
||||
ModuleKey: "glossary",
|
||||
|
||||
@@ -2,9 +2,9 @@ package validators
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/validators/editorial_review"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/validators/meaning_reversal_review"
|
||||
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
|
||||
@@ -30,7 +30,7 @@ const (
|
||||
|
||||
type BuiltInValidatorDefinition struct {
|
||||
Key string
|
||||
Build func() (contracts.Validator, error)
|
||||
Build func() (frameworkvalidators.Validator, error)
|
||||
}
|
||||
|
||||
type Registry struct {
|
||||
@@ -64,7 +64,7 @@ func (r *Registry) Lookup(key string) (BuiltInValidatorDefinition, bool) {
|
||||
return def, ok
|
||||
}
|
||||
|
||||
func (r *Registry) MustBuild(key string) (contracts.Validator, error) {
|
||||
func (r *Registry) MustBuild(key string) (frameworkvalidators.Validator, error) {
|
||||
if r == nil {
|
||||
return nil, fmt.Errorf("validator registry is nil")
|
||||
}
|
||||
|
||||
@@ -2,12 +2,12 @@ package validators
|
||||
|
||||
import (
|
||||
"context"
|
||||
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/validators/editorial_review"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/validators/meaning_reversal_review"
|
||||
@@ -50,7 +50,7 @@ func TestBuiltInValidatorPackagesConstruct(t *testing.T) {
|
||||
type validatorCtor struct {
|
||||
name string
|
||||
key string
|
||||
build func() (contracts.Validator, error)
|
||||
build func() (frameworkvalidators.Validator, error)
|
||||
wantClass validatormetadata.ExecutionClass
|
||||
}
|
||||
cases := []validatorCtor{
|
||||
@@ -117,7 +117,7 @@ func TestRegistryProtectedTermsUsesNonGlossaryStageBehavior(t *testing.T) {
|
||||
t.Fatalf("must build protected_terms: %v", err)
|
||||
}
|
||||
|
||||
req := contracts.ValidationRequest{
|
||||
req := frameworkvalidators.Request{
|
||||
ModuleKey: "glossary",
|
||||
Glossary: []glossary.Entry{{Collection: "party", ID: "jesters", Name: "Jesters", Category: "faction", Description: "Protected"}},
|
||||
CandidateProposal: []proposals.EnrichedCorrectionProposal{
|
||||
@@ -152,7 +152,7 @@ func TestResolveBuiltInChainUsesGlossaryStageProtectedTermsBehavior(t *testing.T
|
||||
t.Fatalf("resolve glossary chain: %v", err)
|
||||
}
|
||||
|
||||
var v contracts.Validator
|
||||
var v frameworkvalidators.Validator
|
||||
for _, candidate := range chain {
|
||||
if candidate.Name() == KeyProtectedTerms {
|
||||
v = candidate
|
||||
@@ -163,7 +163,7 @@ func TestResolveBuiltInChainUsesGlossaryStageProtectedTermsBehavior(t *testing.T
|
||||
t.Fatalf("expected protected_terms validator in glossary chain")
|
||||
}
|
||||
|
||||
req := contracts.ValidationRequest{
|
||||
req := frameworkvalidators.Request{
|
||||
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Jesters"}}},
|
||||
ReplacementPolicy: proposals.ReplacementPolicyReplaceAll,
|
||||
ModuleKey: "glossary",
|
||||
|
||||
@@ -1,14 +1,13 @@
|
||||
package spoken_form_plausibility
|
||||
|
||||
import (
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
|
||||
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
|
||||
validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata"
|
||||
)
|
||||
|
||||
const Key = "spoken_form_plausibility"
|
||||
|
||||
func New() (contracts.Validator, error) {
|
||||
func New() (frameworkvalidators.Validator, error) {
|
||||
v, err := frameworkvalidators.NewLLMBackedValidator(Key, frameworkvalidators.LLMValidatorTypeSpokenFormPlausibility)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
Reference in New Issue
Block a user