diff --git a/docs/internal/validators.md b/docs/internal/validators.md index 8808e67..8257581 100644 --- a/docs/internal/validators.md +++ b/docs/internal/validators.md @@ -53,9 +53,17 @@ assessment is unavailable; an unavailable assessment alone does not. Revised candidates run the applicable checks again. A zero repair budget retains the original shortlist flow. -Decision cardinality is enforced inside Audita: each candidate proposal must -receive exactly one outcome per validator it reaches. The runner still rejects -duplicate, missing, or unknown identities in an internal validator result. +The shared validator port lives in `internal/framework/validators`. Instances +may run concurrently across sections and can keep synchronized state. Request +data is borrowed for the duration of the call. Copy data before modifying or +retaining it. + +The runner checks decision identity and disposition before allowing a candidate +to advance. A normal return supplies one decision per submitted candidate. A +fatal return may provide a checked subset as evidence. Contradictory approval +and disposition are a fatal contract failure. The recorded disposition governs +approval, and each record carries the effective execution class, including +instance overrides, for initial validation and revalidation. ## LLM Validator Batching @@ -89,8 +97,11 @@ Reconciliation makes no semantic retry or extra model call; guided recovery rema [deferred](../roadmap/future.md#domain-aware-response-recovery). Debug request metadata records `proposal_indices` in local-reference order -(reference 1 is the first entry). Raw decoded responses and ignored indices use -local references; reconciled decisions and unavailable indices use internal +(reference 1 is the first entry). Decoding preserves the supplied numeric +tokens, including integral decimal or exponent spellings. Reconciliation +checks the batch range before converting to a local integer, so an extra +reference beyond the native integer range can be ignored. Raw decoded responses +and ignored indices use local references. Reconciled decisions use internal proposal indices. Batch token estimates use the same locally numbered payload as prompt encoding. diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 224247f..a116532 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -355,6 +355,13 @@ beside an out-of-range reference. Cover equivalent numeric spellings, contradictory internal results, and class override from execution through the record supplied to Stage 17. Update [validator internals](../internal/validators.md). +**Complete.** Validators share one port and applicability owner. The runner +rejects contradictory decisions and records the effective class on initial and +repair validation evidence. Controlled provider responses verified exact +numeric decoding, equivalent spellings, duplicate and missing decisions, and +a usable decision alongside a reference above the native integer range. The +CLI build and full offline suite passed. + ## Stage 10 — Reduce validation context work Audit #36. diff --git a/internal/core/jsonnumber/integer.go b/internal/core/jsonnumber/integer.go new file mode 100644 index 0000000..59c02a5 --- /dev/null +++ b/internal/core/jsonnumber/integer.go @@ -0,0 +1,82 @@ +// Package jsonnumber converts exact integral JSON tokens without float rounding. +package jsonnumber + +import ( + "strconv" + "strings" +) + +// Int accepts lexical integers and equivalent decimal and exponent spellings +// in the native integer range. value must be a valid JSON number token. +func Int(value string) (int, bool) { + if integer, err := strconv.Atoi(value); err == nil { + return integer, true + } + digits, ok := integralDecimal(value) + if !ok { + return 0, false + } + integer, err := strconv.Atoi(digits) + return integer, err == nil +} + +// PositiveIndex accepts only references in [1, maximum]. Comparison with the +// requested range precedes conversion to the local integer index. +func PositiveIndex(value string, maximum int) (int, bool) { + if strings.ContainsAny(value, ".eE") { + var ok bool + if value, ok = integralDecimal(value); !ok { + return 0, false + } + } + if maximum < 1 || value == "" || value[0] < '1' || value[0] > '9' { + return 0, false + } + limit := strconv.Itoa(maximum) + if len(value) > len(limit) || (len(value) == len(limit) && value > limit) { + return 0, false + } + index, err := strconv.Atoi(value) + return index, err == nil +} + +func integralDecimal(value string) (string, bool) { + if value == "" || (value[0] != '-' && (value[0] < '0' || value[0] > '9')) || !strings.ContainsAny(value, ".eE") { + return "", false + } + digits, negative := strings.CutPrefix(value, "-") + var exponentText string + if i := strings.IndexAny(digits, "eE"); i >= 0 { + exponentText, digits = digits[i+1:], digits[:i] + } + whole, fraction, _ := strings.Cut(digits, ".") + digits = strings.TrimLeft(whole+fraction, "0") + if digits == "" { + return "0", true + } + var exponent int + if exponentText != "" { + var err error + exponent, err = strconv.Atoi(exponentText) + if err != nil || exponent < -len(digits) || exponent > len(fraction)+19 { + return "", false + } + } + shift := exponent - len(fraction) + if shift < 0 { + // A negative shift is integral only if every discarded digit is zero. + if -shift > len(digits)-len(strings.TrimRight(digits, "0")) { + return "", false + } + digits = digits[:len(digits)+shift] + } else { + if len(digits)+shift > 19 { + return "", false + } + digits += strings.Repeat("0", shift) + } + if negative { + digits = "-" + digits + } + return digits, true +} diff --git a/internal/core/schema/transcript.go b/internal/core/schema/transcript.go index 272e6dd..92890cf 100644 --- a/internal/core/schema/transcript.go +++ b/internal/core/schema/transcript.go @@ -6,8 +6,8 @@ import ( "fmt" "math" "slices" - "strconv" - "strings" + + "gitea.maximumdirect.net/eric/audita/v3/internal/core/jsonnumber" ) type SourceSegment struct { @@ -114,60 +114,13 @@ func extractSegments(raw []byte) (json.RawMessage, error) { type sourceID int func (id *sourceID) UnmarshalJSON(raw []byte) error { - spelling := string(raw) - if value, err := strconv.Atoi(spelling); err == nil { - *id = sourceID(value) - return nil - } - if value, ok := integralDecimal(spelling); ok { + if value, ok := jsonnumber.Int(string(raw)); ok { *id = sourceID(value) return nil } return json.Unmarshal(raw, (*int)(id)) } -func integralDecimal(value string) (int, bool) { - if value == "" || (value[0] != '-' && (value[0] < '0' || value[0] > '9')) || !strings.ContainsAny(value, ".eE") { - return 0, false - } - digits, negative := strings.CutPrefix(value, "-") - var exponentText string - if i := strings.IndexAny(digits, "eE"); i >= 0 { - exponentText, digits = digits[i+1:], digits[:i] - } - whole, fraction, _ := strings.Cut(digits, ".") - digits = strings.TrimLeft(whole+fraction, "0") - if digits == "" { - return 0, true - } - var exponent int - if exponentText != "" { - var err error - exponent, err = strconv.Atoi(exponentText) - if err != nil || exponent < -len(digits) || exponent > len(fraction)+19 { - return 0, false - } - } - shift := exponent - len(fraction) - if shift < 0 { - // A negative shift is integral only if every discarded digit is zero. - if -shift > len(digits)-len(strings.TrimRight(digits, "0")) { - return 0, false - } - digits = digits[:len(digits)+shift] - } else { - if len(digits)+shift > 19 { - return 0, false - } - digits += strings.Repeat("0", shift) - } - if negative { - digits = "-" + digits - } - integer, err := strconv.Atoi(digits) - return integer, err == nil -} - func validateSourceSegments(segments []SourceSegment) error { for i, s := range segments { segLabel := segmentLabel(i, s.ID) diff --git a/internal/framework/contracts/chunk_proposal_composition_test.go b/internal/framework/contracts/chunk_proposal_composition_test.go index f71f4df..94ea13f 100644 --- a/internal/framework/contracts/chunk_proposal_composition_test.go +++ b/internal/framework/contracts/chunk_proposal_composition_test.go @@ -9,6 +9,7 @@ import ( "gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators" ) // testChunkProposalHarness is a test-only helper that composes existing @@ -77,7 +78,7 @@ func (m deterministicFakeModule) ReplacementPolicy() proposals.ReplacementPolicy return m.replacementPolicy } -func (m deterministicFakeModule) Validators() []Validator { return nil } +func (m deterministicFakeModule) Validators() []validators.Validator { return nil } func (m deterministicFakeModule) Propose(ctx context.Context, req ProposalRequest) (ProposalResult, error) { _ = ctx diff --git a/internal/framework/contracts/contracts.go b/internal/framework/contracts/contracts.go index c6c6e44..d4aa8ed 100644 --- a/internal/framework/contracts/contracts.go +++ b/internal/framework/contracts/contracts.go @@ -22,16 +22,10 @@ type StructuredLLMClient = structuredoutput.Client type TranscriptModule interface { Key() string ReplacementPolicy() proposals.ReplacementPolicy - Validators() []Validator + Validators() []validators.Validator Propose(ctx context.Context, req ProposalRequest) (ProposalResult, error) } -// Validator evaluates candidate proposals and returns one decision per proposal index. -type Validator interface { - Name() string - Validate(ctx context.Context, req ValidationRequest) (validators.Result, error) -} - type StructuredCompletionRequest = structuredoutput.Request type StructuredCompletionResponse = structuredoutput.Response @@ -98,9 +92,6 @@ type ProposalResult struct { Operations []outcomes.Operation `json:"operations,omitempty"` } -// ValidationRequest is the input to validator execution. -type ValidationRequest = validators.Request - // ResolveModuleRunSpecs derives isolated consumer views from resolved settings. func ResolveModuleRunSpecs(cfg *config.Config) []ModuleRunSpec { if cfg == nil { diff --git a/internal/framework/contracts/contracts_test.go b/internal/framework/contracts/contracts_test.go index 116e23c..61076ad 100644 --- a/internal/framework/contracts/contracts_test.go +++ b/internal/framework/contracts/contracts_test.go @@ -30,7 +30,7 @@ type fakeValidator struct{} func (f *fakeValidator) Name() string { return "fake-validator" } -func (f *fakeValidator) Validate(ctx context.Context, req ValidationRequest) (validators.Result, error) { +func (f *fakeValidator) Validate(ctx context.Context, req validators.Request) (validators.Result, error) { _ = ctx decisions := make([]validators.Decision, len(req.CandidateProposal)) for i, proposal := range req.CandidateProposal { @@ -47,8 +47,8 @@ func (f *fakeModule) ReplacementPolicy() proposals.ReplacementPolicy { return proposals.ReplacementPolicyRequireUnique } -func (f *fakeModule) Validators() []Validator { - return []Validator{&fakeValidator{}} +func (f *fakeModule) Validators() []validators.Validator { + return []validators.Validator{&fakeValidator{}} } func (f *fakeModule) Propose(ctx context.Context, req ProposalRequest) (ProposalResult, error) { @@ -63,7 +63,7 @@ func (f *fakeModule) Propose(ctx context.Context, req ProposalRequest) (Proposal func TestInterfaceContractsCompileWithFakes(t *testing.T) { var _ StructuredLLMClient = (*fakeLLMClient)(nil) - var _ Validator = (*fakeValidator)(nil) + var _ validators.Validator = (*fakeValidator)(nil) var _ TranscriptModule = (*fakeModule)(nil) module := &fakeModule{} diff --git a/internal/framework/modules/registry_test.go b/internal/framework/modules/registry_test.go index 47f340c..de0f465 100644 --- a/internal/framework/modules/registry_test.go +++ b/internal/framework/modules/registry_test.go @@ -3,6 +3,7 @@ package modules import ( "context" "errors" + frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators" "strings" "testing" @@ -20,7 +21,7 @@ func (m noopModule) Key() string { return m.key } func (m noopModule) ReplacementPolicy() proposals.ReplacementPolicy { return proposals.ReplacementPolicyRequireUnique } -func (m noopModule) Validators() []contracts.Validator { return nil } +func (m noopModule) Validators() []frameworkvalidators.Validator { return nil } func (m noopModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) { _ = ctx _ = req diff --git a/internal/framework/processreport/partial_validation_test.go b/internal/framework/processreport/partial_validation_test.go index 88dee3c..00fdd0b 100644 --- a/internal/framework/processreport/partial_validation_test.go +++ b/internal/framework/processreport/partial_validation_test.go @@ -2,7 +2,9 @@ package processreport import ( "context" + "encoding/json" "errors" + "strconv" "strings" "testing" @@ -16,13 +18,15 @@ import ( "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators" ) -type partialModule struct{ validator contracts.Validator } +type partialModule struct{ validator validators.Validator } func (m partialModule) Key() string { return "grammar" } func (m partialModule) ReplacementPolicy() proposals.ReplacementPolicy { return proposals.ReplacementPolicyRequireUnique } -func (m partialModule) Validators() []contracts.Validator { return []contracts.Validator{m.validator} } +func (m partialModule) Validators() []validators.Validator { + return []validators.Validator{m.validator} +} func (m partialModule) ModuleForSpec(contracts.ModuleRunSpec) (contracts.TranscriptModule, error) { return m, nil } @@ -55,7 +59,7 @@ func (c *interruptedValidationClient) CompleteStructured(_ context.Context, req *target.(*validators.LLMValidationResponse) = validators.LLMValidationResponse{} return structuredoutput.Response{}, nil } - *target.(*validators.LLMValidationResponse) = validators.LLMValidationResponse{Validations: []validators.LLMValidationDecision{{CorrectionIndex: index, Approved: index == 1, Reason: "completed assessment"}}} + *target.(*validators.LLMValidationResponse) = validators.LLMValidationResponse{Validations: []validators.LLMValidationDecision{{CorrectionIndex: json.Number(strconv.Itoa(index)), Approved: index == 1, Reason: "completed assessment"}}} return structuredoutput.Response{}, nil } diff --git a/internal/framework/runner/cancellation_test.go b/internal/framework/runner/cancellation_test.go index f98318a..203ea86 100644 --- a/internal/framework/runner/cancellation_test.go +++ b/internal/framework/runner/cancellation_test.go @@ -104,7 +104,7 @@ func TestValidatorFailureCancelsActiveAndPendingProviderWork(t *testing.T) { }, &out) return validators.Result{}, err }} - module := fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{validator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { + module := fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{validator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "hello", CorrectedText: "Hello"}}, nil }} sections := []chunking.Section{} diff --git a/internal/framework/runner/feedback.go b/internal/framework/runner/feedback.go index eb95d71..c835744 100644 --- a/internal/framework/runner/feedback.go +++ b/internal/framework/runner/feedback.go @@ -1,48 +1,19 @@ package runner import ( - "gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog" - "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators" - validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata" ) // feedbackCandidates submits only candidates for which the validator has usable evidence. -func feedbackCandidates(input validateSectionCandidatesInput, validator contracts.Validator, noEffect map[int]bool, evidence contracts.ValidationRequest) []proposals.EnrichedCorrectionProposal { +func feedbackCandidates(input validateSectionCandidatesInput, validatorName string, modelDriven bool, noEffect map[int]bool, evidence validators.Request) []proposals.EnrichedCorrectionProposal { var result []proposals.EnrichedCorrectionProposal for _, candidate := range input.SectionEnriched { if noEffect[candidate.ProposalIndex] { continue } - usableSpan := candidate.OriginalText != "" - target := candidate.TargetSegmentID > 0 - var applicable bool - switch validator.Name() { - case "proposal_shape": - applicable = true - case "no_effect": - applicable = usableSpan - case "original_text_presence": - applicable = usableSpan && target - case "protected_terms": - applicable = usableSpan - if input.Spec.ModuleKey == modulecatalog.KeyGlossary || input.Spec.ModuleKey == modulecatalog.KeySpokenWord { - preview := evidence.Preview(candidate) - applicable = preview.Applicable || preview.SkipReason == proposals.SkipReasonEmptyResultingText - } - case "non_empty_resulting_segment": - preview := evidence.Preview(candidate) - applicable = preview.Applicable || preview.SkipReason == proposals.SkipReasonEmptyResultingText - default: - if validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked { - applicable = evidence.Preview(candidate).Applicable - } else { - applicable = true - } - } - if applicable { + if evidence.AcceptsIndependentCandidate(validatorName, modelDriven, candidate) { result = append(result, candidate) } } diff --git a/internal/framework/runner/feedback_test.go b/internal/framework/runner/feedback_test.go index 087195a..a93e92a 100644 --- a/internal/framework/runner/feedback_test.go +++ b/internal/framework/runner/feedback_test.go @@ -25,7 +25,7 @@ func TestFeedbackCollectsIndependentFindingsAndUsableSemanticInput(t *testing.T) return proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: target, OriginalText: original, CorrectedText: replacement}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index}} } var seen []int - semantic := validatormetadata.Wrap(fakeValidator{name: "semantic", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { + semantic := validatormetadata.Wrap(fakeValidator{name: "semantic", validateF: func(req validators.Request) (validators.Result, error) { var decisions []validators.Decision for _, p := range req.CandidateProposal { seen = append(seen, p.ProposalIndex) @@ -33,7 +33,7 @@ func TestFeedbackCollectsIndependentFindingsAndUsableSemanticInput(t *testing.T) } return validators.Result{Decisions: decisions}, nil }}, validatormetadata.ExecutionClassLLMBacked) - input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh teh cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, semantic}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate(0, 1, "cat", "dog"), candidate(1, 0, "teh", "the"), candidate(2, 1, "teh", "the"), candidate(3, 1, "cat", "cat")}} + input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh teh cat"}}}, Validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, semantic}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate(0, 1, "cat", "dog"), candidate(1, 0, "teh", "the"), candidate(2, 1, "teh", "the"), candidate(3, 1, "cat", "cat")}} result, err := validateSectionCandidates(context.Background(), input) if err != nil { t.Fatal(err) @@ -58,14 +58,14 @@ func TestFeedbackKeepsUnavailableAndZeroBudgetShortlists(t *testing.T) { for _, budget := range []int{0, 1} { t.Run(fmt.Sprint(budget), func(t *testing.T) { var later int - first := fakeValidator{name: "first", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { + first := fakeValidator{name: "first", validateF: func(req validators.Request) (validators.Result, error) { var decisions []validators.Decision for _, p := range req.CandidateProposal { decisions = append(decisions, validators.Decision{ProposalIndex: p.ProposalIndex, Disposition: validators.DispositionUnavailable, ReasonCode: "unavailable"}) } return validators.Result{Decisions: decisions}, nil }} - second := fakeValidator{name: "second", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { + second := fakeValidator{name: "second", validateF: func(req validators.Request) (validators.Result, error) { later = len(req.CandidateProposal) var decisions []validators.Decision for _, p := range req.CandidateProposal { @@ -73,7 +73,7 @@ func TestFeedbackKeepsUnavailableAndZeroBudgetShortlists(t *testing.T) { } return validators.Result{Decisions: decisions}, nil }} - input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{first, second}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}} + input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{first, second}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}} result, err := validateSectionCandidates(context.Background(), input) if err != nil { t.Fatal(err) @@ -95,8 +95,8 @@ func TestRepairFindingsUseSafeUnavailableEvidence(t *testing.T) { func TestRevalidationArtifactsUseProposalAndAttemptCoordinates(t *testing.T) { client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{ - {Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}, - {Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}, + {Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}, + {Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}, }} validator, err := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview) if err != nil { @@ -108,7 +108,7 @@ func TestRevalidationArtifactsUseProposalAndAttemptCoordinates(t *testing.T) { var paths []string for _, index := range []int{2, 3} { candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: index}} - input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar_1", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validator}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, ValidationLLMClient: client, Interactions: sink, SectionIndex: §ion, RepairProposalIndex: &index, RepairAttempt: &attempt} + input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar_1", Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{validator}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, ValidationLLMClient: client, Interactions: sink, SectionIndex: §ion, RepairProposalIndex: &index, RepairAttempt: &attempt} result, err := validateSectionCandidates(context.Background(), input) if err != nil { t.Fatal(err) @@ -149,11 +149,11 @@ func TestRevalidationArtifactsUseProposalAndAttemptCoordinates(t *testing.T) { func TestFeedbackReportsProtectionAndBlankResult(t *testing.T) { identity := glossary.Identity{Collection: "party", ID: "jesters"} - protected := fakeValidator{name: "protected_terms", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { + protected := fakeValidator{name: "protected_terms", validateF: func(req validators.Request) (validators.Result, error) { return validators.ValidateProtectedTerms(req) }} candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "Jesters", CorrectedText: ""}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}} - input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "spoken_word", ProtectedTerms: []glossary.Term{{Spelling: "Jesters", Identities: []glossary.Identity{identity}}}, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Jesters"}}}, Validators: []contracts.Validator{protected, validators.NonEmptyResultingSegmentValidator{}}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}} + input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "spoken_word", ProtectedTerms: []glossary.Term{{Spelling: "Jesters", Identities: []glossary.Identity{identity}}}, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Jesters"}}}, Validators: []validators.Validator{protected, validators.NonEmptyResultingSegmentValidator{}}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}} result, err := validateSectionCandidates(context.Background(), input) if err != nil { t.Fatal(err) diff --git a/internal/framework/runner/ordered_workflow_test.go b/internal/framework/runner/ordered_workflow_test.go index 7d5ee0e..d14b687 100644 --- a/internal/framework/runner/ordered_workflow_test.go +++ b/internal/framework/runner/ordered_workflow_test.go @@ -2,6 +2,7 @@ package runner import ( "context" + frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators" "testing" "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" @@ -27,7 +28,7 @@ func (m orderedModule) Key() string { return m.spec.ModuleKey } func (m orderedModule) ReplacementPolicy() proposals.ReplacementPolicy { return proposals.ReplacementPolicyRequireUnique } -func (m orderedModule) Validators() []contracts.Validator { return nil } +func (m orderedModule) Validators() []frameworkvalidators.Validator { return nil } func (m orderedModule) Propose(_ context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) { m.seen <- req original, corrected := "party", "Party" diff --git a/internal/framework/runner/repair_test.go b/internal/framework/runner/repair_test.go index df26acc..ec15e2b 100644 --- a/internal/framework/runner/repair_test.go +++ b/internal/framework/runner/repair_test.go @@ -24,6 +24,7 @@ import ( "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators" + validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata" ) type repairClient struct { @@ -52,7 +53,7 @@ func TestRepairBranchesRetargetAndPreserveOtherApprovals(t *testing.T) { {CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 4}}, {CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 0, OriginalText: "x", CorrectedText: "y"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 5}}, } - input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}, {ID: 2, Text: "uh hello"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.NonEmptyResultingSegmentValidator{}}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{}} + input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}, {ID: 2, Text: "uh hello"}}}, Validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.NonEmptyResultingSegmentValidator{}}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{}} initial, err := validateSectionCandidates(context.Background(), input) if err != nil { t.Fatal(err) @@ -92,7 +93,7 @@ func TestRepairRequestCarriesLiteralPreview(t *testing.T) { Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: tc.policy, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 81, Text: tc.segment}}}, - Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.NonEmptyResultingSegmentValidator{}, rejectUntilRepair()}, + Validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validators.NonEmptyResultingSegmentValidator{}, rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}, } @@ -131,7 +132,7 @@ func TestRepairRequestRefreshesPreviewAfterRetarget(t *testing.T) { `{"proposal_ref":1,"decision":{"action":"withdraw","correction":null}}`, }} var validatedIDs []int - reject := fakeValidator{name: "editorial_review", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { + reject := fakeValidator{name: "editorial_review", validateF: func(req validators.Request) (validators.Result, error) { validatedIDs = append(validatedIDs, req.CandidateProposal[0].TargetSegmentID) return validators.Result{Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Disposition: validators.DispositionRejected, ReasonCode: "llm_rejected", Message: "The context does not support this edit."}}}, nil }} @@ -139,9 +140,10 @@ func TestRepairRequestRefreshesPreviewAfterRetarget(t *testing.T) { Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 2}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 81, Text: "cat"}, {ID: 104, Text: "uh hello"}}}, - Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, reject}, + Validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, validatormetadata.Wrap(reject, validatormetadata.ExecutionClassDeterministic)}, SectionEnriched: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 81, OriginalText: "cat", CorrectedText: "dog"}}}, Exchange: &structuredoutput.Exchange{}, + Interactions: llm.NewDiagnosticsWriter(t.TempDir()), } initial, err := validateSectionCandidates(context.Background(), input) if err != nil { @@ -154,6 +156,15 @@ func TestRepairRequestRefreshesPreviewAfterRetarget(t *testing.T) { if len(validatedIDs) != 2 || validatedIDs[0] != 81 || validatedIDs[1] != 104 { t.Fatalf("validated segment IDs: %v", validatedIDs) } + for _, decisions := range [][]ValidatorDecisionRecord{initial.decisions, branches[0].Attempts[0].Decisions} { + if len(decisions) == 0 { + t.Fatal("missing decision evidence") + } + record := decisions[len(decisions)-1] + if record.ValidatorName != "editorial_review" || record.ExecutionClass != validatormetadata.ExecutionClassDeterministic { + t.Fatalf("lost execution class override: %+v", record) + } + } for i, want := range []struct{ before, after string }{{"cat", "dog"}, {"uh hello", "hello"}} { var payload proposal_generation.RepairRequest if err := json.Unmarshal([]byte(client.calls[i].Variables["RepairRequestJSON"]), &payload); err != nil { @@ -183,7 +194,7 @@ func TestRepairBranchTerminalOutcomes(t *testing.T) { t.Run(tc.name, func(t *testing.T) { client := &repairClient{t: t, outputs: tc.outputs, err: tc.err} candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}} - input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: tc.budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}} + input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: tc.budget}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}} initial, err := validateSectionCandidates(context.Background(), input) if err != nil { t.Fatal(err) @@ -211,7 +222,7 @@ func TestRunnerAppliesRevisionsInOriginalOrder(t *testing.T) { client := &repairClient{t: t, outputs: []string{`{"proposal_ref":2,"decision":{"action":"revise","correction":{"id":1,"original_text":"cat","corrected_text":"fox"}}}`, `{"proposal_ref":3,"decision":{"action":"revise","correction":{"id":1,"original_text":"bird","corrected_text":"owl"}}}`, }} - module := repairModule{outcomeModule{fakeModule: fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, rejectUntilRepair(0)}}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) { + module := repairModule{outcomeModule{fakeModule: fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NoEffectValidator{}, validators.OriginalTextPresenceValidator{}, rejectUntilRepair(0)}}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) { return contracts.ProposalResult{Proposals: candidates, SegmentReferences: []int{1, 1, 1}, Exchange: &structuredoutput.Exchange{}}, nil }}} runner := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"grammar": module}}) @@ -244,7 +255,7 @@ func TestRepairRevalidatesEarlierApprovalAndRetainsUnrelatedApprovalOnFailure(t }{{"revision", false}, {"provider failure", true}} { t.Run(tc.name, func(t *testing.T) { calls := 0 - check := fakeValidator{name: "custom", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { + check := fakeValidator{name: "custom", validateF: func(req validators.Request) (validators.Result, error) { var decisions []validators.Decision for _, candidate := range req.CandidateProposal { calls++ @@ -256,7 +267,7 @@ func TestRepairRevalidatesEarlierApprovalAndRetainsUnrelatedApprovalOnFailure(t if tc.failure { client.err = &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonProviderFailed, Message: "failed"} } - input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat bird"}}}, Validators: []contracts.Validator{check, rejectUntilRepair(0)}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{}} + input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat bird"}}}, Validators: []validators.Validator{check, rejectUntilRepair(0)}, SectionEnriched: candidates, Exchange: &structuredoutput.Exchange{}} initial, err := validateSectionCandidates(context.Background(), input) if err != nil { t.Fatal(err) @@ -279,7 +290,7 @@ func TestRepairRevalidatesEarlierApprovalAndRetainsUnrelatedApprovalOnFailure(t func TestRepairInterruptionRetainsAttempt(t *testing.T) { client := &repairClient{t: t, err: &structuredoutput.Failure{Class: structuredoutput.FailureCanceled, Reason: outcomes.ReasonCanceled, Message: "canceled", Cause: context.Canceled}} candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}} - input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}} + input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}} initial, err := validateSectionCandidates(context.Background(), input) if err != nil { t.Fatal(err) @@ -297,7 +308,7 @@ func TestRepairArtifactsAreUnique(t *testing.T) { {CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "fox"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 1}}, } section := 4 - input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar_1", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: candidates, SectionIndex: §ion, Exchange: &structuredoutput.Exchange{}, Interactions: llm.NewDiagnosticsWriter(t.TempDir())} + input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar_1", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{rejectUntilRepair()}, SectionEnriched: candidates, SectionIndex: §ion, Exchange: &structuredoutput.Exchange{}, Interactions: llm.NewDiagnosticsWriter(t.TempDir())} initial, err := validateSectionCandidates(context.Background(), input) if err != nil { t.Fatal(err) @@ -355,7 +366,7 @@ func TestRepairArtifactFailureDoesNotChangeOutcome(t *testing.T) { } sink := llm.NewDiagnosticsWriter(path) candidate := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0}} - input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []contracts.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}, Interactions: sink} + input := validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{InstanceName: "grammar", RepairClient: client, Settings: config.Stage{LLM: config.LLM{CorrectionRepairAttempts: 1}}}, Policy: proposals.ReplacementPolicyRequireUnique, WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "cat"}}}, Validators: []validators.Validator{rejectUntilRepair()}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, Exchange: &structuredoutput.Exchange{}, Interactions: sink} initial, err := validateSectionCandidates(context.Background(), input) if err != nil { t.Fatal(err) @@ -382,8 +393,8 @@ func TestNoEffectAndUnavailableOnlyDoNotStartRepair(t *testing.T) { } // rejectUntilRepair supplies a semantic rejection at the external validator seam. -func rejectUntilRepair(preserved ...int) contracts.Validator { - return fakeValidator{name: "editorial_review", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { +func rejectUntilRepair(preserved ...int) validators.Validator { + return fakeValidator{name: "editorial_review", validateF: func(req validators.Request) (validators.Result, error) { var decisions []validators.Decision for _, p := range req.CandidateProposal { approved := req.RepairAttempt != nil diff --git a/internal/framework/runner/runner.go b/internal/framework/runner/runner.go index d5edb61..c37ae05 100644 --- a/internal/framework/runner/runner.go +++ b/internal/framework/runner/runner.go @@ -69,13 +69,14 @@ type WorkUnitOutcome struct { } type ValidatorDecisionRecord struct { - ValidatorName string `json:"validator_name"` - ProposalIndex int `json:"proposal_index"` - Approved bool `json:"approved"` - Disposition validators.DecisionDisposition `json:"disposition"` - ReasonCode string `json:"reason_code"` - Message string `json:"message"` - DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"` + ExecutionClass validatormetadata.ExecutionClass `json:"-"` + ValidatorName string `json:"validator_name"` + ProposalIndex int `json:"proposal_index"` + Approved bool `json:"approved"` + Disposition validators.DecisionDisposition `json:"disposition"` + ReasonCode string `json:"reason_code"` + Message string `json:"message"` + DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"` } type ValidatorRejectedChange struct { @@ -650,7 +651,7 @@ type validateSectionCandidatesInput struct { Policy proposals.ReplacementPolicy WorkingTranscript *schema.Transcript SectionIndex *int - Validators []contracts.Validator + Validators []validators.Validator SectionEnriched []proposals.EnrichedCorrectionProposal ValidationLLMClient contracts.StructuredLLMClient Interactions structuredoutput.InteractionSink @@ -673,7 +674,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida eligible := append([]proposals.EnrichedCorrectionProposal(nil), input.SectionEnriched...) feedback := input.Spec.Settings.LLM.CorrectionRepairAttempts > 0 var blocked, noEffect map[int]bool - evidence := contracts.ValidationRequest{WorkingTranscript: input.WorkingTranscript, Section: input.Section, ReplacementPolicy: input.Policy, IndependentEvidence: feedback} + evidence := validators.Request{WorkingTranscript: input.WorkingTranscript, Section: input.Section, ReplacementPolicy: input.Policy, IndependentEvidence: feedback} if feedback { blocked, noEffect = make(map[int]bool), make(map[int]bool) evidence.Previews = make(map[int]proposals.SegmentPreviewResult) @@ -686,11 +687,12 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida if err := ctx.Err(); err != nil { return validateSectionCandidatesResult{decisions: decisions, rejected: rejected, operations: operations, filtered: filtered}, err } + class := validatormetadata.ClassOf(validator) submitted := eligible if feedback { - submitted = feedbackCandidates(input, validator, noEffect, evidence) + submitted = feedbackCandidates(input, validator.Name(), class == validatormetadata.ExecutionClassLLMBacked, noEffect, evidence) } - vResult, err := validator.Validate(ctx, contracts.ValidationRequest{ + vResult, err := validator.Validate(ctx, validators.Request{ StageIndex: &input.Spec.Index, StageID: input.Spec.ID, WorkingTranscript: input.WorkingTranscript, Section: evidence.Section, Previews: evidence.Previews, @@ -728,17 +730,20 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida byIndex[p.ProposalIndex] = p } - for _, d := range vResult.Decisions { - if feedback && validator.Name() == "original_text_presence" && d.Approved { - preview := evidence.Preview(byIndex[d.ProposalIndex]) - if preview.SkipReason == proposals.SkipReasonAmbiguousOriginal { - d.Approved = false - d.Disposition = validators.DispositionRejected - d.ReasonCode = string(proposals.SkipReasonAmbiguousOriginal) - d.Message = "original_text matches multiple locations in the target segment" - } + dispositions := make([]validators.DecisionDisposition, len(vResult.Decisions)) + for i, decision := range vResult.Decisions { + disposition, contractErr := decision.EffectiveDisposition() + if contractErr != nil { + return validateSectionCandidatesResult{ + decisions: decisions, rejected: rejected, operations: operations, filtered: filtered, + }, fmt.Errorf("validator %q returned inconsistent decisions: %w", validator.Name(), errors.Join(err, contractErr)) } - if feedback && !d.Approved { + dispositions[i] = disposition + } + for i, d := range vResult.Decisions { + disposition := dispositions[i] + approved := disposition == validators.DispositionApproved + if feedback && !approved { blocked[d.ProposalIndex] = true if d.ReasonCode == validators.ReasonNoEffect { noEffect[d.ProposalIndex] = true @@ -746,18 +751,19 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida } decisions = append(decisions, ValidatorDecisionRecord{ ValidatorName: validator.Name(), + ExecutionClass: class, ProposalIndex: d.ProposalIndex, - Approved: d.Approved, - Disposition: d.Disposition, + Approved: approved, + Disposition: disposition, ReasonCode: d.ReasonCode, Message: d.Message, DiagnosticArtifactPath: d.DiagnosticArtifactPath, }) - if d.Approved { + if approved { nextEligible = append(nextEligible, byIndex[d.ProposalIndex]) continue } - if d.Disposition != validators.DispositionUnavailable { + if disposition != validators.DispositionUnavailable { if reason, ok := validators.DefectiveProposalReason(d.ReasonCode); ok { filtered = append(filtered, outcomes.FilteredProposal{ProposalIndex: d.ProposalIndex, Reason: reason, Count: 1}) } @@ -771,7 +777,7 @@ func validateSectionCandidates(ctx context.Context, input validateSectionCandida TargetSegmentID: p.TargetSegmentID, OriginalText: p.OriginalText, CorrectedText: p.CorrectedText, - Disposition: d.Disposition, + Disposition: disposition, ReasonCode: d.ReasonCode, Message: d.Message, }) @@ -882,11 +888,13 @@ func attributeApplyFiltering(sections []chunking.Section, approved []proposals.E } } -func reorderValidatorsForPipeline(in []contracts.Validator) ([]contracts.Validator, map[string]int) { - deterministic := make([]contracts.Validator, 0, len(in)) - llmBacked := make([]contracts.Validator, 0, len(in)) +func reorderValidatorsForPipeline(in []validators.Validator) ([]validators.Validator, map[string]int) { + deterministic := make([]validators.Validator, 0, len(in)) + llmBacked := make([]validators.Validator, 0, len(in)) for _, validator := range in { - if validatormetadata.ClassOf(validator) == validatormetadata.ExecutionClassLLMBacked { + class := validatormetadata.ClassOf(validator) + validator = validatormetadata.Wrap(validator, class) + if class == validatormetadata.ExecutionClassLLMBacked { llmBacked = append(llmBacked, validator) continue } diff --git a/internal/framework/runner/runner_test.go b/internal/framework/runner/runner_test.go index 90a7592..83f3784 100644 --- a/internal/framework/runner/runner_test.go +++ b/internal/framework/runner/runner_test.go @@ -43,7 +43,7 @@ func (f fakeFactory) ModuleForSpec(spec contracts.ModuleRunSpec) (contracts.Tran type fakeModule struct { key string policy proposals.ReplacementPolicy - validators []contracts.Validator + validators []validators.Validator proposeF func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) } @@ -52,6 +52,33 @@ type outcomeModule struct { proposeResult func(contracts.ProposalRequest) (contracts.ProposalResult, error) } +func TestRunnerStopsOnContradictoryValidatorDecision(t *testing.T) { + candidate := proposals.EnrichedCorrectionProposal{ + CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, + ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 7}, + } + contractFailure := errors.New("validator also returned a runtime error") + for _, cause := range []error{nil, contractFailure} { + contradictory := fakeValidator{name: "invalid", validateF: func(validators.Request) (validators.Result, error) { + return validators.Result{Decisions: []validators.Decision{{ + ProposalIndex: 7, Approved: true, Disposition: validators.DispositionUnavailable, + }}}, cause + }} + later := fakeValidator{name: "later", validateF: func(validators.Request) (validators.Result, error) { + t.Fatal("candidate advanced past a contradictory decision") + return validators.Result{}, nil + }} + result, err := validateSectionCandidates(context.Background(), validateSectionCandidatesInput{ + Validators: []validators.Validator{validators.ProposalShapeValidator{}, contradictory, later}, + SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}, + }) + if err == nil || !strings.Contains(err.Error(), "contradictory") || (cause != nil && !errors.Is(err, cause)) || + len(result.approved) != 0 || len(result.decisions) != 1 || result.decisions[0].ValidatorName != "proposal_shape" { + t.Fatalf("contradictory result = %+v (%v)", result, err) + } + } +} + func TestRunnerRecoverableProposalFailureAllowsLaterModule(t *testing.T) { failed := outcomeModule{fakeModule: fakeModule{key: "first", policy: proposals.ReplacementPolicyRequireUnique}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) { return contracts.ProposalResult{Operations: []outcomes.Operation{{Reference: outcomes.Reference{ModuleKey: "first", ModuleInstance: "first", Kind: outcomes.OperationProposal}, FailureReason: outcomes.ReasonTransportFailed}}}, nil @@ -74,7 +101,7 @@ func TestRunnerRecoverableProposalFailureAllowsLaterModule(t *testing.T) { func TestRunnerOnlyFullyValidatedCandidatesApplyAndRecoveriesMerge(t *testing.T) { secondSaw := make([]int, 0) - partial := fakeValidator{name: "partial", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { + partial := fakeValidator{name: "partial", validateF: func(req validators.Request) (validators.Result, error) { if len(req.CandidateProposal) != 2 { t.Fatalf("expected two candidates after shape filtering, got %d", len(req.CandidateProposal)) } @@ -86,13 +113,13 @@ func TestRunnerOnlyFullyValidatedCandidatesApplyAndRecoveriesMerge(t *testing.T) {Reference: outcomes.Reference{Kind: outcomes.OperationValidation}, FailureReason: outcomes.ReasonProviderFailed}, }}, nil }} - last := fakeValidator{name: "last", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { + last := fakeValidator{name: "last", validateF: func(req validators.Request) (validators.Result, error) { for _, candidate := range req.CandidateProposal { secondSaw = append(secondSaw, candidate.ProposalIndex) } return validators.Result{ValidatorName: "last", Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: true, Disposition: validators.DispositionApproved}}}, nil }} - module := outcomeModule{fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{validators.ProposalShapeValidator{}, partial, last}}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) { + module := outcomeModule{fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{validators.ProposalShapeValidator{}, partial, last}}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) { return contracts.ProposalResult{Proposals: []proposals.CorrectionProposal{ {TargetSegmentID: 0, OriginalText: "bad", CorrectedText: "x"}, {TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}, @@ -120,10 +147,10 @@ func TestRunnerOnlyFullyValidatedCandidatesApplyAndRecoveriesMerge(t *testing.T) func TestFatalValidatorNeverReturnsUncheckedCandidates(t *testing.T) { origin := errors.New("validator setup failed") proposal := proposals.EnrichedCorrectionProposal{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}, ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 7}} - broken := fakeValidator{name: "broken", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { + broken := fakeValidator{name: "broken", validateF: func(req validators.Request) (validators.Result, error) { return validators.Result{Operations: []outcomes.Operation{{Reference: outcomes.Reference{Kind: outcomes.OperationValidation}, FailureReason: outcomes.ReasonSetupFailure}}}, origin }} - result, err := validateSectionCandidates(context.Background(), validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "m", InstanceName: "m"}, Validators: []contracts.Validator{broken}, SectionEnriched: []proposals.EnrichedCorrectionProposal{proposal}}) + result, err := validateSectionCandidates(context.Background(), validateSectionCandidatesInput{Spec: contracts.ModuleRunSpec{ModuleKey: "m", InstanceName: "m"}, Validators: []validators.Validator{broken}, SectionEnriched: []proposals.EnrichedCorrectionProposal{proposal}}) if !errors.Is(err, origin) || len(result.approved) != 0 || len(result.operations) != 1 { t.Fatalf("fatal result must retain facts without approving candidates: result=%+v err=%v", result, err) } @@ -131,7 +158,7 @@ func TestFatalValidatorNeverReturnsUncheckedCandidates(t *testing.T) { func TestCandidateCaptureIsExplicitAndIncludesFilteredProposals(t *testing.T) { module := fakeModule{key: "grammar", policy: proposals.ReplacementPolicyRequireUnique, - validators: []contracts.Validator{validators.ProposalShapeValidator{}}, + validators: []validators.Validator{validators.ProposalShapeValidator{}}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 0, OriginalText: "bad", CorrectedText: "good"}}, nil }} @@ -160,7 +187,7 @@ func (m outcomeModule) Propose(_ context.Context, req contracts.ProposalRequest) func (m fakeModule) Key() string { return m.key } func (m fakeModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy } -func (m fakeModule) Validators() []contracts.Validator { return m.validators } +func (m fakeModule) Validators() []validators.Validator { return m.validators } func (m fakeModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) { if m.proposeF == nil { return contracts.ProposalResult{}, nil @@ -171,11 +198,11 @@ func (m fakeModule) Propose(ctx context.Context, req contracts.ProposalRequest) type fakeValidator struct { name string - validateF func(req contracts.ValidationRequest) (validators.Result, error) + validateF func(req validators.Request) (validators.Result, error) } func (v fakeValidator) Name() string { return v.name } -func (v fakeValidator) Validate(ctx context.Context, req contracts.ValidationRequest) (validators.Result, error) { +func (v fakeValidator) Validate(ctx context.Context, req validators.Request) (validators.Result, error) { _ = ctx return v.validateF(req) } @@ -191,19 +218,19 @@ func (v classifiedFakeValidator) ExecutionClass() validatormetadata.ExecutionCla func TestReorderValidatorsDeterministicBeforeLLMBacked(t *testing.T) { llm := classifiedFakeValidator{ - fakeValidator: fakeValidator{name: "llm", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { + fakeValidator: fakeValidator{name: "llm", validateF: func(req validators.Request) (validators.Result, error) { return validators.Result{ValidatorName: "llm"}, nil }}, class: validatormetadata.ExecutionClassLLMBacked, } deterministic := classifiedFakeValidator{ - fakeValidator: fakeValidator{name: "deterministic", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { + fakeValidator: fakeValidator{name: "deterministic", validateF: func(req validators.Request) (validators.Result, error) { return validators.Result{ValidatorName: "deterministic"}, nil }}, class: validatormetadata.ExecutionClassDeterministic, } - ordered, _ := reorderValidatorsForPipeline([]contracts.Validator{llm, deterministic}) + ordered, _ := reorderValidatorsForPipeline([]validators.Validator{llm, deterministic}) if len(ordered) != 2 { t.Fatalf("expected 2 validators, got %d", len(ordered)) } @@ -213,17 +240,17 @@ func TestReorderValidatorsDeterministicBeforeLLMBacked(t *testing.T) { } func TestReorderValidatorsDefaultsUnclassifiedToDeterministic(t *testing.T) { - unclassified := fakeValidator{name: "plain", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { + unclassified := fakeValidator{name: "plain", validateF: func(req validators.Request) (validators.Result, error) { return validators.Result{ValidatorName: "plain"}, nil }} llm := classifiedFakeValidator{ - fakeValidator: fakeValidator{name: "llm", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { + fakeValidator: fakeValidator{name: "llm", validateF: func(req validators.Request) (validators.Result, error) { return validators.Result{ValidatorName: "llm"}, nil }}, class: validatormetadata.ExecutionClassLLMBacked, } - ordered, _ := reorderValidatorsForPipeline([]contracts.Validator{llm, unclassified}) + ordered, _ := reorderValidatorsForPipeline([]validators.Validator{llm, unclassified}) if len(ordered) != 2 { t.Fatalf("expected 2 validators, got %d", len(ordered)) } @@ -553,7 +580,7 @@ func TestRunnerValidationStartsBeforeAllSectionProposalsComplete(t *testing.T) { validator := fakeValidator{ name: "capture", - validateF: func(req contracts.ValidationRequest) (validators.Result, error) { + validateF: func(req validators.Request) (validators.Result, error) { if len(req.CandidateProposal) == 1 && req.CandidateProposal[0].TargetSegmentID == 1 { select { case validatorStarted <- struct{}{}: @@ -577,7 +604,7 @@ func TestRunnerValidationStartsBeforeAllSectionProposalsComplete(t *testing.T) { "m": fakeModule{ key: "m", policy: proposals.ReplacementPolicyRequireUnique, - validators: []contracts.Validator{validator}, + validators: []validators.Validator{validator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { seg := req.WorkingTranscript.Segments[0] if req.Section != nil && req.Section.Index == 1 { @@ -630,7 +657,7 @@ func (c *captureContractStructuredClient) CompleteStructured(ctx context.Context if target, ok := out.(*validators.LLMValidationResponse); ok { *target = validators.LLMValidationResponse{ Validations: []validators.LLMValidationDecision{ - {CorrectionIndex: 1, Approved: true, Reason: "ok"}, + {CorrectionIndex: "1", Approved: true, Reason: "ok"}, }, } } @@ -662,7 +689,7 @@ func (c *stageAwareStructuredClient) CompleteStructured(ctx context.Context, req target := out.(*validators.LLMValidationResponse) *target = validators.LLMValidationResponse{ Validations: []validators.LLMValidationDecision{ - {CorrectionIndex: 1, Approved: true, Reason: "ok"}, + {CorrectionIndex: "1", Approved: true, Reason: "ok"}, }, } return contracts.StructuredCompletionResponse{}, nil @@ -734,7 +761,7 @@ func TestRunnerRepeatedModuleInstanceNames(t *testing.T) { func TestRunnerValidatorApprovedProposalApplied(t *testing.T) { transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}} - allowAll := fakeValidator{name: "allow", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { + allowAll := fakeValidator{name: "allow", validateF: func(req validators.Request) (validators.Result, error) { decisions := make([]validators.Decision, len(req.CandidateProposal)) for i, p := range req.CandidateProposal { decisions[i] = validators.Decision{ProposalIndex: p.ProposalIndex, Approved: true, ReasonCode: validators.ReasonApproved, Message: "approved"} @@ -745,7 +772,7 @@ func TestRunnerValidatorApprovedProposalApplied(t *testing.T) { "m": fakeModule{ key: "m", policy: proposals.ReplacementPolicyRequireUnique, - validators: []contracts.Validator{allowAll}, + validators: []validators.Validator{allowAll}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil }, @@ -766,7 +793,7 @@ func TestRunnerValidatorApprovedProposalApplied(t *testing.T) { func TestRunnerValidatorRejectedProposalNotApplied(t *testing.T) { transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}} - rejectAll := fakeValidator{name: "reject", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { + rejectAll := fakeValidator{name: "reject", validateF: func(req validators.Request) (validators.Result, error) { decisions := make([]validators.Decision, len(req.CandidateProposal)) for i, p := range req.CandidateProposal { decisions[i] = validators.Decision{ProposalIndex: p.ProposalIndex, Approved: false, ReasonCode: validators.ReasonNoEffect, Message: "rejected"} @@ -774,7 +801,7 @@ func TestRunnerValidatorRejectedProposalNotApplied(t *testing.T) { return validators.Result{ValidatorName: "reject", Decisions: decisions}, nil }} r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ - "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{rejectAll}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { + "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{rejectAll}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil }}, }}) @@ -792,7 +819,7 @@ func TestRunnerValidatorRejectedProposalNotApplied(t *testing.T) { func TestRunnerMultipleValidatorsRunInOrderAndFilterSurvivors(t *testing.T) { transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "one two"}}} - first := fakeValidator{name: "first", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { + first := fakeValidator{name: "first", validateF: func(req validators.Request) (validators.Result, error) { if len(req.CandidateProposal) != 2 { t.Fatalf("expected first validator to see 2 candidates, got %d", len(req.CandidateProposal)) } @@ -804,14 +831,14 @@ func TestRunnerMultipleValidatorsRunInOrderAndFilterSurvivors(t *testing.T) { }, }, nil }} - second := fakeValidator{name: "second", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { + second := fakeValidator{name: "second", validateF: func(req validators.Request) (validators.Result, error) { if len(req.CandidateProposal) != 1 { t.Fatalf("expected second validator to see only survivors, got %d", len(req.CandidateProposal)) } return validators.Result{ValidatorName: "second", Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: true, ReasonCode: validators.ReasonApproved, Message: "ok"}}}, nil }} r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ - "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{first, second}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { + "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{first, second}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{ {TargetSegmentID: 1, OriginalText: "one", CorrectedText: "ONE"}, {TargetSegmentID: 1, OriginalText: "two", CorrectedText: "TWO"}, @@ -833,11 +860,11 @@ func TestRunnerValidatorCardinalityErrorStopsPipelineWithPartialProgress(t *test good := fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil }} - badValidator := fakeValidator{name: "bad", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { + badValidator := fakeValidator{name: "bad", validateF: func(req validators.Request) (validators.Result, error) { // Missing one decision triggers cardinality error. return validators.Result{ValidatorName: "bad", Decisions: nil}, nil }} - bad := fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{badValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { + bad := fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{badValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}}, nil }} r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{"m1": good, "m2": bad}}) @@ -856,11 +883,11 @@ func TestRunnerValidatorCardinalityErrorStopsPipelineWithPartialProgress(t *test func TestRunnerApplicationSkipAfterValidatorApprovalReported(t *testing.T) { transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "word word"}}} - allow := fakeValidator{name: "allow", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { + allow := fakeValidator{name: "allow", validateF: func(req validators.Request) (validators.Result, error) { return validators.Result{ValidatorName: "allow", Decisions: []validators.Decision{{ProposalIndex: req.CandidateProposal[0].ProposalIndex, Approved: true, ReasonCode: validators.ReasonApproved, Message: "ok"}}}, nil }} r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ - "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{allow}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { + "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{allow}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "word", CorrectedText: "term"}}, nil }}, }}) @@ -880,7 +907,7 @@ func TestRunnerApplyConflictAfterTwoInitiallyValidDeletions(t *testing.T) { transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "a b"}}} r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "m": outcomeModule{ - fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{validators.ProposalShapeValidator{}, validators.NonEmptyResultingSegmentValidator{}}}, + fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{validators.ProposalShapeValidator{}, validators.NonEmptyResultingSegmentValidator{}}}, proposeResult: func(req contracts.ProposalRequest) (contracts.ProposalResult, error) { return contracts.ProposalResult{ Proposals: []proposals.CorrectionProposal{ @@ -945,14 +972,14 @@ func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contr } func TestRunnerLLMValidatorApprovalApplied(t *testing.T) { - client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}}} + client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}}} llmValidator, err := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility) if err != nil { t.Fatalf("NewLLMBackedValidator: %v", err) } r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ - "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { + "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil }}, }}) @@ -971,10 +998,10 @@ func TestRunnerLLMValidatorApprovalApplied(t *testing.T) { } func TestRunnerLLMValidatorRejectionPreventsApplication(t *testing.T) { - client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: false, Reason: "reject"}}}}} + client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: false, Reason: "reject"}}}}} llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility) r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ - "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { + "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil }}, }}) @@ -996,13 +1023,13 @@ func TestRunnerLLMValidatorRejectionPreventsApplication(t *testing.T) { } func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress(t *testing.T) { - client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Reason: "bad index"}}}}} + client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "99", Approved: true, Reason: "bad index"}}}}} llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility) r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ "m1": fakeModule{key: "m1", policy: proposals.ReplacementPolicyRequireUnique, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}}, nil }}, - "m2": fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { + "m2": fakeModule{key: "m2", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}}, nil }}, }}) @@ -1028,13 +1055,13 @@ func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress( func TestRunnerReconciledSurvivorsCompleteLaterValidation(t *testing.T) { client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{ - {CorrectionIndex: 1, Approved: true, Reason: "ok"}, - {CorrectionIndex: 3, Approved: true, Reason: "ok"}, - {CorrectionIndex: 99, Approved: true, Reason: "unexpected"}, + {CorrectionIndex: "1", Approved: true, Reason: "ok"}, + {CorrectionIndex: "3", Approved: true, Reason: "ok"}, + {CorrectionIndex: "99", Approved: true, Reason: "unexpected"}, }}}} first, _ := validators.NewLLMBackedValidator("editorial_review", validators.LLMValidatorTypeEditorialReview) var laterIndices []int - later := fakeValidator{name: "later_review", validateF: func(req contracts.ValidationRequest) (validators.Result, error) { + later := fakeValidator{name: "later_review", validateF: func(req validators.Request) (validators.Result, error) { decisions := make([]validators.Decision, 0, len(req.CandidateProposal)) for _, candidate := range req.CandidateProposal { laterIndices = append(laterIndices, candidate.ProposalIndex) @@ -1047,7 +1074,7 @@ func TestRunnerReconciledSurvivorsCompleteLaterValidation(t *testing.T) { } return validators.Result{ValidatorName: "later_review", Decisions: decisions}, nil }} - module := outcomeModule{fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{first, validatormetadata.Wrap(later, validatormetadata.ExecutionClassLLMBacked)}}, proposeResult: func(contracts.ProposalRequest) (contracts.ProposalResult, error) { + module := outcomeModule{fakeModule: fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{first, validatormetadata.Wrap(later, validatormetadata.ExecutionClassLLMBacked)}}, proposeResult: func(contracts.ProposalRequest) (contracts.ProposalResult, error) { return contracts.ProposalResult{Proposals: []proposals.CorrectionProposal{ {TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the"}, {TargetSegmentID: 1, OriginalText: "cat", CorrectedText: "dog"}, @@ -1077,7 +1104,7 @@ func TestRunnerLLMValidatorMissingDecisionRejectsBatch(t *testing.T) { client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{}}}} llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility) r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ - "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { + "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil }}, }}) @@ -1094,12 +1121,12 @@ func TestRunnerLLMValidatorMissingDecisionRejectsBatch(t *testing.T) { func TestRunnerLLMValidatorDuplicateDecisionRejectsBatch(t *testing.T) { client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{ - {CorrectionIndex: 1, Approved: true, Reason: "ok"}, - {CorrectionIndex: 1, Approved: false, Reason: "dup"}, + {CorrectionIndex: "1", Approved: true, Reason: "ok"}, + {CorrectionIndex: "1", Approved: false, Reason: "dup"}, }}}} llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility) r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ - "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { + "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil }}, }}) @@ -1120,8 +1147,8 @@ func (lenEstimator) EstimateTokens(text string) int { return len(text) } func TestRunnerLLMValidatorBatching(t *testing.T) { client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{ - {Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}, - {Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}, + {Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}, + {Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}, }} llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility) llmValidator.SetTokenEstimator(lenEstimator{}) @@ -1129,7 +1156,7 @@ func TestRunnerLLMValidatorBatching(t *testing.T) { cfg := runnerTestConfig() cfg.Validation.MaxPromptTokens = 260 r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ - "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyReplaceAll, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { + "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyReplaceAll, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{ {TargetSegmentID: 1, OriginalText: "alpha", CorrectedText: strings.Repeat("B", 40)}, {TargetSegmentID: 1, OriginalText: "gamma", CorrectedText: strings.Repeat("D", 40)}, @@ -1154,12 +1181,12 @@ func TestRunnerLLMValidatorBatching(t *testing.T) { } func TestRunnerLLMValidatorDiagnosticsWritten(t *testing.T) { - client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "supported by context"}}}}} + client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "supported by context"}}}}} llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility) cfg := runnerTestConfig() diagDir := t.TempDir() r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{ - "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { + "m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []validators.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return []proposals.CorrectionProposal{{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters"}}, nil }}, }}) @@ -1208,7 +1235,7 @@ type proposalGenerationModule struct { func (m proposalGenerationModule) Key() string { return m.key } func (m proposalGenerationModule) ReplacementPolicy() proposals.ReplacementPolicy { return m.policy } -func (m proposalGenerationModule) Validators() []contracts.Validator { return nil } +func (m proposalGenerationModule) Validators() []validators.Validator { return nil } func (m proposalGenerationModule) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) { result, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{ ModuleKey: req.RunSpec.ModuleKey, @@ -1294,10 +1321,10 @@ func TestFatalPartialDecisionsRejectUnknownAndDuplicateIdentity(t *testing.T) { {{ProposalIndex: 8, Approved: true}}, {{ProposalIndex: 7, Approved: true}, {ProposalIndex: 7, Approved: true}}, } { - v := fakeValidator{name: "broken", validateF: func(contracts.ValidationRequest) (validators.Result, error) { + v := fakeValidator{name: "broken", validateF: func(validators.Request) (validators.Result, error) { return validators.Result{Decisions: decisions}, origin }} - result, err := validateSectionCandidates(context.Background(), validateSectionCandidatesInput{Validators: []contracts.Validator{v}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}}) + result, err := validateSectionCandidates(context.Background(), validateSectionCandidatesInput{Validators: []validators.Validator{v}, SectionEnriched: []proposals.EnrichedCorrectionProposal{candidate}}) if !errors.Is(err, origin) || len(result.decisions) != 0 || len(result.approved) != 0 { t.Fatalf("invalid partial evidence accepted or origin lost: %+v %v", result, err) } diff --git a/internal/framework/validators/deterministic.go b/internal/framework/validators/deterministic.go index d7e3a5a..931a693 100644 --- a/internal/framework/validators/deterministic.go +++ b/internal/framework/validators/deterministic.go @@ -37,6 +37,7 @@ func (v OriginalTextPresenceValidator) Name() string { return "original_text_pre func (v OriginalTextPresenceValidator) Validate(_ context.Context, req Request) (Result, error) { section := req.sectionLookup() + req.Section = section decisions := make([]Decision, 0, len(req.CandidateProposal)) for _, c := range req.CandidateProposal { @@ -49,6 +50,10 @@ func (v OriginalTextPresenceValidator) Validate(_ context.Context, req Request) decisions = append(decisions, rejection(c.ProposalIndex, ReasonMissingOriginalText, "original_text was not found in current segment text")) continue } + if req.IndependentEvidence && req.Preview(c).SkipReason == proposals.SkipReasonAmbiguousOriginal { + decisions = append(decisions, rejection(c.ProposalIndex, string(proposals.SkipReasonAmbiguousOriginal), "original_text matches multiple locations in the target segment")) + continue + } decisions = append(decisions, approval(c.ProposalIndex)) } if err := EnforceDecisionCardinality(req.CandidateProposal, decisions); err != nil { diff --git a/internal/framework/validators/llm_context_test.go b/internal/framework/validators/llm_context_test.go index 7e589ec..715d5e1 100644 --- a/internal/framework/validators/llm_context_test.go +++ b/internal/framework/validators/llm_context_test.go @@ -25,15 +25,15 @@ func TestValidationContextCarriesAdjacentEvidenceAndRelevantDefinitions(t *testi {ID: "lyra", Name: "Lyra", Description: "The host"}, {ID: "unrelated", Name: "Unrelated", Description: "Not mentioned"}, } - payload, immediate := BuildLLMValidationRequest("spoken_form_plausibility", LLMValidatorTypeSpokenFormPlausibility, req) - if len(immediate) != 0 || len(payload.Items) != 1 { + payload, immediate := buildValidationItems(req) + if len(immediate) != 0 || len(payload) != 1 { t.Fatalf("payload: %+v immediate: %+v", payload, immediate) } - item := payload.Items[0] + item := payload[0] if len(item.NeighboringSegments) != 2 || item.NeighboringSegments[0].Position != "preceding" || item.NeighboringSegments[1].Position != "following" || len(item.GlossaryEntries) != 2 || item.GlossaryEntries[0].Name != "Jesters" || item.GlossaryEntries[1].Name != "Lyra" { t.Fatalf("context: %+v", item) } - variables, err := BuildValidationVariables(payload.Items, "") + variables, err := BuildValidationVariables(payload, "") if err != nil { t.Fatal(err) } @@ -41,7 +41,7 @@ func TestValidationContextCarriesAdjacentEvidenceAndRelevantDefinitions(t *testi if err := json.Unmarshal([]byte(variables["PayloadJSON"]), &rendered); err != nil { t.Fatal(err) } - if strings.Contains(variables["PayloadJSON"], `"segment_id":`) || rendered[0].CorrectionIndex != 1 || payload.Items[0].CorrectionIndex != 7 { + if strings.Contains(variables["PayloadJSON"], `"segment_id":`) || rendered[0].CorrectionIndex != 1 || payload[0].CorrectionIndex != 7 { t.Fatalf("leaked internal identity or mutated request: %s", variables["PayloadJSON"]) } if rendered[0].Speaker != "Guest" || len(rendered[0].NeighboringSegments) != 2 || rendered[0].NeighboringSegments[0].Speaker != "Host" || rendered[0].GlossaryEntries[0].Description != "A traveling troupe" { @@ -51,15 +51,15 @@ func TestValidationContextCarriesAdjacentEvidenceAndRelevantDefinitions(t *testi req.WorkingTranscript.Segments = req.WorkingTranscript.Segments[1:2] req.WorkingTranscript.Segments[0].Speaker = "" req.Glossary = nil - isolated, _ := BuildLLMValidationRequest("editorial_review", LLMValidatorTypeEditorialReview, req) - if isolated.Items[0].Speaker != "" || len(isolated.Items[0].NeighboringSegments) != 0 || len(isolated.Items[0].GlossaryEntries) != 0 { + isolated, _ := buildValidationItems(req) + if isolated[0].Speaker != "" || len(isolated[0].NeighboringSegments) != 0 || len(isolated[0].GlossaryEntries) != 0 { t.Fatalf("unexpected external context: %+v", isolated) } } func TestValidationContextFitsBudgetWithoutDiscardingCandidate(t *testing.T) { estimator := chunking.NewSimpleTokenEstimator() - base := LLMValidationItem{CorrectionIndex: 7, SegmentID: 1, OriginalText: "teh", CorrectedText: "the", OriginalSegmentText: "teh cat", CorrectedSegmentText: "the cat"} + base := LLMValidationItem{CorrectionIndex: 7, OriginalText: "teh", CorrectedText: "the", OriginalSegmentText: "teh cat", CorrectedSegmentText: "the cat"} required, err := estimateBatchTokens(estimator, []LLMValidationItem{base}) if err != nil { t.Fatal(err) @@ -89,12 +89,12 @@ func TestValidationContextSelectsGlossaryForEachReplacement(t *testing.T) { req := makeReq([]proposals.EnrichedCorrectionProposal{mk(7, "gestures", "Jesters"), mk(11, "gestures", "Actors")}) req.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Those gestures are back."}}} req.Glossary = []glossary.Entry{{ID: "actors", Name: "Actors"}, {ID: "jesters", Name: "Jesters"}} - payload, immediate := BuildLLMValidationRequest("editorial_review", LLMValidatorTypeEditorialReview, req) - if len(immediate) != 0 || len(payload.Items) != 2 { + payload, immediate := buildValidationItems(req) + if len(immediate) != 0 || len(payload) != 2 { t.Fatalf("payload: %+v immediate: %+v", payload, immediate) } for i, name := range []string{"Jesters", "Actors"} { - entries := payload.Items[i].GlossaryEntries + entries := payload[i].GlossaryEntries if len(entries) != 1 || entries[0].Name != name { t.Fatalf("candidate %d context: %+v", i, entries) } diff --git a/internal/framework/validators/llm_integration_test.go b/internal/framework/validators/llm_integration_test.go index fb9eae1..e861b0e 100644 --- a/internal/framework/validators/llm_integration_test.go +++ b/internal/framework/validators/llm_integration_test.go @@ -15,6 +15,7 @@ import ( "gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" + "gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput" ck "go.maximumdirect.net/configkit" ) @@ -50,28 +51,7 @@ func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) { } return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil }) - dir := t.TempDir() - main := filepath.Join(dir, "config.yml") - if err := os.WriteFile(main, []byte("version: 3\n"), 0600); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte("profiles:\n - {id: base, endpoint: 'https://example.invalid/v1', model: local-model, api_key_required: false}\n"), 0600); err != nil { - t.Fatal(err) - } - loaded, err := config.Load(ck.Input{Source: ck.File(main)}, "") - if err != nil { - t.Fatal(err) - } - runtime, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport}) - if err != nil { - t.Fatal(err) - } - settings := loaded.Config.LLM - settings.TransientRetries, settings.RepairAttempts = 0, 1 - client, err := runtime.NewRoleClient("validation", "base", settings, llm.CallHooks{}) - if err != nil { - t.Fatal(err) - } + client := testValidationClient(t, transport) validator, err := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview) if err != nil { t.Fatal(err) @@ -91,3 +71,98 @@ func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) { }) } } + +// The transport supplies provider responses; PromptKit performs structured +// validation and Audita decodes the same response through the role client. +func testValidationClient(t *testing.T, transport http.RoundTripper) structuredoutput.Client { + t.Helper() + dir := t.TempDir() + main := filepath.Join(dir, "config.yml") + if err := os.WriteFile(main, []byte("version: 3\n"), 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte("profiles:\n - {id: base, endpoint: 'https://example.invalid/v1', model: local-model, api_key_required: false}\n"), 0600); err != nil { + t.Fatal(err) + } + loaded, err := config.Load(ck.Input{Source: ck.File(main)}, "") + if err != nil { + t.Fatal(err) + } + runtime, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport}) + if err != nil { + t.Fatal(err) + } + settings := loaded.Config.LLM + settings.TransientRetries, settings.RepairAttempts = 0, 1 + client, err := runtime.NewRoleClient("validation", "base", settings, llm.CallHooks{}) + if err != nil { + t.Fatal(err) + } + return client +} + +func TestValidationDecodesExactReferencesBeforeReconciliation(t *testing.T) { + for _, tc := range []struct { + name, response string + indices []int + want []DecisionDisposition + reasons []string + failure outcomes.ReasonCode + }{ + { + name: "usable decision beside extra", + response: `{"validations":[{"correction_index":1.00e0,"approved":true,"reason":"accept"},{"correction_index":9223372036854775808,"approved":true,"reason":"extra"}]}`, + indices: []int{7}, + want: []DecisionDisposition{DispositionApproved}, + reasons: []string{ReasonApproved}, + }, + { + name: "usable siblings beside duplicate and omission", + response: `{"validations":[{"correction_index":1e0,"approved":true,"reason":"accept"},{"correction_index":2,"approved":true,"reason":"duplicate"},{"correction_index":20e-1,"approved":false,"reason":"duplicate"},{"correction_index":4.000,"approved":false,"reason":"reject"},{"correction_index":9223372036854775808,"approved":true,"reason":"extra"}]}`, + indices: []int{7, 11, 14, 23}, + want: []DecisionDisposition{DispositionApproved, DispositionUnavailable, DispositionUnavailable, DispositionRejected}, + reasons: []string{ReasonApproved, ReasonDuplicateValidationDecision, ReasonMissingValidationDecision, "llm_rejected"}, + failure: outcomes.ReasonValidationDecisionsInvalid, + }, + } { + t.Run(tc.name, func(t *testing.T) { + calls := 0 + client := testValidationClient(t, validationTransport(func(req *http.Request) (*http.Response, error) { + calls++ + content, _ := json.Marshal(tc.response) + body := fmt.Sprintf(`{"choices":[{"message":{"content":%s}}]}`, content) + return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: req}, nil + })) + var candidates []proposals.EnrichedCorrectionProposal + for _, index := range tc.indices { + candidates = append(candidates, mk(index, "gestures", "Jesters")) + } + validator, err := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview) + if err != nil { + t.Fatal(err) + } + sink := &captureValidationDiagnosticsWriter{} + req := makeReq(candidates) + req.LLMClient = client + req.Interactions = sink + result, err := validator.Validate(context.Background(), req) + if err != nil || calls != 1 || len(result.Decisions) != len(tc.want) || len(result.Operations) != 1 { + t.Fatalf("result=%+v calls=%d error=%v", result, calls, err) + } + for i, decision := range result.Decisions { + if decision.ProposalIndex != tc.indices[i] || decision.Disposition != tc.want[i] || decision.ReasonCode != tc.reasons[i] { + t.Fatalf("decision %d: %+v", i, decision) + } + } + operation := result.Operations[0] + if operation.FailureReason != tc.failure || (tc.failure == "" && (len(operation.RecoveryReasons) != 1 || operation.RecoveryReasons[0] != outcomes.RecoveryValidationDecisionsFiltered)) { + t.Fatalf("operation: %+v", operation) + } + payload := sink.lastResponsePayload.(map[string]any) + diagnostics := payload["reconciliation"].(validationReconciliation) + if len(diagnostics.IgnoredIndices) != 1 || diagnostics.IgnoredIndices[0] != "9223372036854775808" || payload["final_content"] != tc.response { + t.Fatalf("reference evidence changed: %+v", payload) + } + }) + } +} diff --git a/internal/framework/validators/llm_models.go b/internal/framework/validators/llm_models.go index 699b699..91ae62a 100644 --- a/internal/framework/validators/llm_models.go +++ b/internal/framework/validators/llm_models.go @@ -1,6 +1,8 @@ package validators import ( + "encoding/json" + "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput" ) @@ -21,7 +23,6 @@ type StructuredLLMClient = structuredoutput.Client // BuildValidationVariables substitutes a one-based reference for each batch. type LLMValidationItem struct { CorrectionIndex int `json:"correction_index"` - SegmentID int `json:"-"` Speaker string `json:"speaker"` OriginalText string `json:"original_text"` CorrectedText string `json:"corrected_text"` @@ -32,21 +33,11 @@ type LLMValidationItem struct { GlossaryEntries []glossary.Entry `json:"glossary_entries,omitempty"` } -// LLMValidationRequest is the canonical request model for LLM-backed validators. -type LLMValidationRequest struct { - ValidatorName string `json:"validator_name"` - ValidatorType LLMValidatorType `json:"validator_type"` - ModuleKey string `json:"module_key"` - ModuleInstance string `json:"module_instance"` - ReplacementPolicy string `json:"replacement_policy"` - Items []LLMValidationItem `json:"corrections"` -} - // LLMValidationDecision contains a request-local correction_index, not a proposal index. type LLMValidationDecision struct { - CorrectionIndex int `json:"correction_index"` - Approved bool `json:"approved"` - Reason string `json:"reason"` + CorrectionIndex json.Number `json:"correction_index"` + Approved bool `json:"approved"` + Reason string `json:"reason"` } type LLMValidationResponse struct { diff --git a/internal/framework/validators/llm_partial_test.go b/internal/framework/validators/llm_partial_test.go index 7b23fa4..a73b104 100644 --- a/internal/framework/validators/llm_partial_test.go +++ b/internal/framework/validators/llm_partial_test.go @@ -48,9 +48,9 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) { } failed := &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonTransportFailed, Message: "model transport failed"} client := &scriptedValidationClient{replies: []validationReply{ - {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}}, + {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}}, {completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 1}}, err: failed}, - {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}}, + {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}}, }} validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview) validator.SetTokenEstimator(perItemEstimator{}) @@ -73,8 +73,8 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) { survivors := []proposals.EnrichedCorrectionProposal{candidates[0], candidates[2]} nextClient := &scriptedValidationClient{replies: []validationReply{ - {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}}, - {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}}, + {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}}, + {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}}, }} nextValidator, _ := NewLLMBackedValidator("meaning_reversal_review", LLMValidatorTypeMeaningReversal) nextValidator.SetTokenEstimator(perItemEstimator{}) @@ -101,7 +101,7 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) { func TestValidationDomainInvalidAfterRepairIsAnError(t *testing.T) { client := &scriptedValidationClient{replies: []validationReply{{ - response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Reason: "ok"}}}, + response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "99", Approved: true, Reason: "ok"}}}, completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 2, StructuralRepairs: 1}}, }}} validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview) @@ -116,8 +116,8 @@ func TestValidationDomainInvalidAfterRepairIsAnError(t *testing.T) { func TestValidationExtraDecisionsMergeActualRecoveries(t *testing.T) { client := &scriptedValidationClient{replies: []validationReply{{ response: LLMValidationResponse{Validations: []LLMValidationDecision{ - {CorrectionIndex: 1, Approved: true, Reason: "ok"}, - {CorrectionIndex: 99, Approved: true, Reason: "extra"}, + {CorrectionIndex: "1", Approved: true, Reason: "ok"}, + {CorrectionIndex: "99", Approved: true, Reason: "extra"}, }}, completion: StructuredCompletionResponse{Execution: outcomes.ExecutionCounts{Available: true, ProviderAttempts: 3, TransientRetries: 1, StructuralRepairs: 1}}, }}} @@ -137,9 +137,9 @@ func TestValidationExtraDecisionsMergeActualRecoveries(t *testing.T) { func TestValidationFatalFailureKeepsEarlierFactsAndStops(t *testing.T) { cause := errors.New("broken embedded schema") client := &scriptedValidationClient{replies: []validationReply{ - {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}}, + {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}}, {err: &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "model setup failed", Cause: cause}}, - {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}}, + {response: LLMValidationResponse{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}}, }} validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview) validator.SetTokenEstimator(perItemEstimator{}) diff --git a/internal/framework/validators/llm_validators.go b/internal/framework/validators/llm_validators.go index 56a9c82..d291027 100644 --- a/internal/framework/validators/llm_validators.go +++ b/internal/framework/validators/llm_validators.go @@ -2,6 +2,7 @@ package validators import ( "context" + "encoding/json" "errors" "fmt" "sort" @@ -9,6 +10,7 @@ import ( "gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking" "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" + "gitea.maximumdirect.net/eric/audita/v3/internal/core/jsonnumber" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/stagename" @@ -58,8 +60,8 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, if req.LLMClient == nil { return Result{}, fmt.Errorf("LLM-backed validator %q requires a structured LLM client", v.name) } - validationReq, immediate := BuildLLMValidationRequest(v.name, v.validatorType, req) - if len(validationReq.Items) == 0 { + items, immediate := buildValidationItems(req) + if len(items) == 0 { all := append([]Decision(nil), immediate...) sort.SliceStable(all, func(i, j int) bool { return all[i].ProposalIndex < all[j].ProposalIndex }) if err := EnforceDecisionCardinality(req.CandidateProposal, all); err != nil { @@ -73,7 +75,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, maxTokens = req.Settings.Validation.MaxPromptTokens } - batches, oversized, err := prepareValidationBatches(validationReq.Items, maxTokens, v.estimator) + batches, oversized, err := prepareValidationBatches(items, maxTokens, v.estimator) operations := make([]outcomes.Operation, 0) llmDecisions := make([]Decision, 0) partialResult := func() Result { @@ -239,7 +241,7 @@ func errPayload(err error) any { return map[string]any{"error": err.Error()} } -func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorType, req Request) (LLMValidationRequest, []Decision) { +func buildValidationItems(req Request) ([]LLMValidationItem, []Decision) { items := make([]LLMValidationItem, 0, len(req.CandidateProposal)) immediate := make([]Decision, 0) req.Section = req.sectionLookup() @@ -268,7 +270,6 @@ func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorT neighbors, glossaryEntries := glossary.validationContext(segments, position, preview.CorrectedSegmentText) items = append(items, LLMValidationItem{ CorrectionIndex: p.ProposalIndex, - SegmentID: p.TargetSegmentID, Speaker: seg.Speaker, OriginalText: p.OriginalText, CorrectedText: p.CorrectedText, @@ -280,31 +281,25 @@ func BuildLLMValidationRequest(validatorName string, validatorType LLMValidatorT }) } - return LLMValidationRequest{ - ValidatorName: validatorName, - ValidatorType: validatorType, - ModuleKey: req.ModuleKey, - ModuleInstance: req.ModuleInstance, - ReplacementPolicy: string(req.ReplacementPolicy), - Items: items, - }, immediate + return items, immediate } type validationReconciliation struct { - Decisions []Decision `json:"decisions"` - UnavailableIndices []int `json:"unavailable_indices,omitempty"` - IgnoredIndices []int `json:"ignored_indices,omitempty"` + Decisions []Decision `json:"decisions"` + UnavailableIndices []int `json:"unavailable_indices,omitempty"` + IgnoredIndices []json.Number `json:"ignored_indices,omitempty"` } func reconcileLLMResponse(items []LLMValidationItem, response LLMValidationResponse) validationReconciliation { result := validationReconciliation{Decisions: make([]Decision, 0, len(items))} seen := make(map[int][]LLMValidationDecision, len(items)) for _, d := range response.Validations { - if d.CorrectionIndex < 1 || d.CorrectionIndex > len(items) { + index, ok := jsonnumber.PositiveIndex(d.CorrectionIndex.String(), len(items)) + if !ok { result.IgnoredIndices = append(result.IgnoredIndices, d.CorrectionIndex) continue } - seen[d.CorrectionIndex] = append(seen[d.CorrectionIndex], d) + seen[index] = append(seen[index], d) } for i, item := range items { diff --git a/internal/framework/validators/llm_validators_test.go b/internal/framework/validators/llm_validators_test.go index a98b6ec..6f2a941 100644 --- a/internal/framework/validators/llm_validators_test.go +++ b/internal/framework/validators/llm_validators_test.go @@ -4,6 +4,8 @@ import ( "context" "encoding/json" "errors" + "slices" + "strconv" "strings" "testing" @@ -74,7 +76,7 @@ func mk(index int, orig, corr string) proposals.EnrichedCorrectionProposal { } } -func TestBuildLLMValidationRequestPreservesLiteralDeletionAndSpacing(t *testing.T) { +func TestValidationPayloadPreservesLiteralDeletionAndSpacing(t *testing.T) { for _, tc := range []struct { name, segment, original, replacement, result string }{ @@ -84,27 +86,25 @@ func TestBuildLLMValidationRequestPreservesLiteralDeletionAndSpacing(t *testing. t.Run(tc.name, func(t *testing.T) { req := makeReq([]proposals.EnrichedCorrectionProposal{mk(7, tc.original, tc.replacement)}) req.WorkingTranscript.Segments[0].Text = tc.segment - payload, immediate := BuildLLMValidationRequest("editorial_review", LLMValidatorTypeEditorialReview, req) - if len(immediate) != 0 || len(payload.Items) != 1 { + payload, immediate := buildValidationItems(req) + if len(immediate) != 0 || len(payload) != 1 { t.Fatalf("expected previewable request: %+v, %+v", payload, immediate) } - raw, err := json.Marshal(payload) + variables, err := BuildValidationVariables(payload, "") if err != nil { t.Fatal(err) } - var decoded struct { - ValidatorName string `json:"validator_name"` - Corrections []map[string]any `json:"corrections"` - } + raw := []byte(variables["PayloadJSON"]) + var decoded []map[string]any if err := json.Unmarshal(raw, &decoded); err != nil { t.Fatal(err) } - if decoded.ValidatorName != "editorial_review" || len(decoded.Corrections) != 1 { - t.Fatalf("wrong validator identity or item count: %s", raw) + if len(decoded) != 1 { + t.Fatalf("wrong item count: %s", raw) } - item := decoded.Corrections[0] + item := decoded[0] value, present := item["corrected_text"] - if !present || value != tc.replacement || item["original_text"] != tc.original || item["original_segment_text"] != tc.segment || item["corrected_segment_text"] != tc.result || item["correction_index"] != float64(7) { + if !present || value != tc.replacement || item["original_text"] != tc.original || item["original_segment_text"] != tc.segment || item["corrected_segment_text"] != tc.result || item["correction_index"] != float64(1) { t.Fatalf("literal spans or full segment text lost: %s", raw) } }) @@ -148,8 +148,8 @@ func TestChunkLLMValidationItemsMultipleBatchesStableNoDropNoDup(t *testing.T) { func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) { client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{ - {CorrectionIndex: 1, Approved: true, Reason: ""}, - {CorrectionIndex: 2, Approved: false, Reason: "bad"}, + {CorrectionIndex: "1", Approved: true, Reason: ""}, + {CorrectionIndex: "2", Approved: false, Reason: "bad"}, }}}} v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility) if err != nil { @@ -203,10 +203,10 @@ func TestLLMBackedValidatorProviderMalformedEnvelopePropagates(t *testing.T) { func TestLLMBackedValidatorReconcilesDecisionIdentities(t *testing.T) { approve := func(index int) LLMValidationDecision { - return LLMValidationDecision{CorrectionIndex: index, Approved: true, Reason: "accepted"} + return LLMValidationDecision{CorrectionIndex: json.Number(strconv.Itoa(index)), Approved: true, Reason: "accepted"} } reject := func(index int) LLMValidationDecision { - return LLMValidationDecision{CorrectionIndex: index, Reason: "editorial concern"} + return LLMValidationDecision{CorrectionIndex: json.Number(strconv.Itoa(index)), Reason: "editorial concern"} } type expectedDecision struct { index int @@ -218,13 +218,13 @@ func TestLLMBackedValidatorReconcilesDecisionIdentities(t *testing.T) { entries []LLMValidationDecision want []expectedDecision unavailable []int - ignored []int + ignored []json.Number }{ - {"internal index is not a local reference", []LLMValidationDecision{approve(7), approve(2), reject(3)}, []expectedDecision{{7, false, ReasonMissingValidationDecision}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, []int{7}, []int{7}}, - {"missing and extra", []LLMValidationDecision{approve(1), reject(3), approve(99)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonMissingValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, []int{99}}, + {"internal index is not a local reference", []LLMValidationDecision{approve(7), approve(2), reject(3)}, []expectedDecision{{7, false, ReasonMissingValidationDecision}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, []int{7}, []json.Number{"7"}}, + {"missing and extra", []LLMValidationDecision{approve(1), reject(3), approve(99)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonMissingValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, []json.Number{"99"}}, {"conflicting duplicate", []LLMValidationDecision{approve(1), approve(2), reject(2), reject(3)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonDuplicateValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil}, {"identical duplicate", []LLMValidationDecision{approve(1), approve(2), approve(2), reject(3)}, []expectedDecision{{7, true, ReasonApproved}, {11, false, ReasonDuplicateValidationDecision}, {18, false, "llm_rejected"}}, []int{11}, nil}, - {"extra only", []LLMValidationDecision{approve(99), reject(3), approve(1), approve(99), approve(2)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, []int{99, 99}}, + {"extra only", []LLMValidationDecision{approve(99), reject(3), approve(1), approve(99), approve(2)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, []json.Number{"99", "99"}}, {"empty", nil, []expectedDecision{{7, false, ReasonMissingValidationDecision}, {11, false, ReasonMissingValidationDecision}, {18, false, ReasonMissingValidationDecision}}, []int{7, 11, 18}, nil}, {"reordered", []LLMValidationDecision{reject(3), approve(2), approve(1)}, []expectedDecision{{7, true, ReasonApproved}, {11, true, ReasonApproved}, {18, false, "llm_rejected"}}, nil, nil}, } { @@ -264,7 +264,7 @@ func TestLLMBackedValidatorReconcilesDecisionIdentities(t *testing.T) { t.Fatalf("supplied model decisions lost from diagnostic: %+v", decoded) } reconciliation := payload["reconciliation"].(validationReconciliation) - if !equalIndices(reconciliation.UnavailableIndices, tc.unavailable) || !equalIndices(reconciliation.IgnoredIndices, tc.ignored) || len(reconciliation.Decisions) != len(tc.want) { + if !equalIndices(reconciliation.UnavailableIndices, tc.unavailable) || !slices.Equal(reconciliation.IgnoredIndices, tc.ignored) || len(reconciliation.Decisions) != len(tc.want) { t.Fatalf("diagnostic reconciliation=%+v", reconciliation) } }) @@ -285,7 +285,7 @@ func equalIndices(a, b []int) bool { func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *testing.T) { client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{ - {CorrectionIndex: 1, Approved: true, Reason: "ok"}, + {CorrectionIndex: "1", Approved: true, Reason: "ok"}, }}}} v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility) huge := strings.Repeat("gestures ", 200) @@ -331,7 +331,7 @@ func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *tes func TestLLMBackedValidatorDiagnosticsIncludeSchemaMetadata(t *testing.T) { client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{ - {CorrectionIndex: 1, Approved: true, Reason: "ok"}, + {CorrectionIndex: "1", Approved: true, Reason: "ok"}, }}}} writer := &captureValidationDiagnosticsWriter{} v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility) @@ -370,7 +370,7 @@ func TestEveryValidationTypeSelectsEmbeddedPromptAndVariables(t *testing.T) { } for _, tc := range cases { t.Run(string(tc.kind), func(t *testing.T) { - client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}}} + client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}}} validator, err := NewLLMBackedValidator(string(tc.kind), tc.kind) if err != nil { t.Fatal(err) @@ -395,7 +395,7 @@ func TestEveryValidationTypeSelectsEmbeddedPromptAndVariables(t *testing.T) { func TestValidationDiagnosticFailureDoesNotChangeDecision(t *testing.T) { cause := errors.New("diagnostic storage unavailable") req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")}) - req.LLMClient = &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Reason: "ok"}}}}} + req.LLMClient = &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: "1", Approved: true, Reason: "ok"}}}}} sink := &captureValidationDiagnosticsWriter{err: cause} req.Interactions = sink validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview) diff --git a/internal/framework/validators/models.go b/internal/framework/validators/models.go index 0612346..93b0681 100644 --- a/internal/framework/validators/models.go +++ b/internal/framework/validators/models.go @@ -62,6 +62,25 @@ type Decision struct { DiagnosticArtifactPath string `json:"diagnostic_artifact_path,omitempty"` } +// EffectiveDisposition resolves an omitted disposition from the declared +// approval and detects inconsistent explicit decisions. +func (d Decision) EffectiveDisposition() (DecisionDisposition, error) { + switch d.Disposition { + case "": + if d.Approved { + return DispositionApproved, nil + } + return DispositionRejected, nil + case DispositionApproved, DispositionRejected, DispositionUnavailable: + if d.Approved == (d.Disposition == DispositionApproved) { + return d.Disposition, nil + } + return "", fmt.Errorf("contradictory approval and disposition %q for proposal %d", d.Disposition, d.ProposalIndex) + default: + return "", fmt.Errorf("invalid decision disposition %q for proposal %d", d.Disposition, d.ProposalIndex) + } +} + // DecisionDisposition separates normal editorial rejection from a candidate // excluded because its model operation could not provide a trustworthy decision. type DecisionDisposition string @@ -80,7 +99,10 @@ type Result struct { Operations []outcomes.Operation `json:"operations,omitempty"` } -// Validator is the deterministic runtime validator interface. +// Validator evaluates one section's candidates. An instance may be called +// concurrently across sections and may keep synchronized state. Request data +// is borrowed for the call; inputs must be copied before modification or +// retention. Request.Preview may populate the caller's pass-local cache. type Validator interface { Name() string Validate(ctx context.Context, req Request) (Result, error) diff --git a/internal/framework/validators/section.go b/internal/framework/validators/section.go index b635a1b..34e453d 100644 --- a/internal/framework/validators/section.go +++ b/internal/framework/validators/section.go @@ -54,3 +54,21 @@ func (r Request) Preview(candidate proposals.EnrichedCorrectionProposal) proposa } return preview } + +// AcceptsIndependentCandidate decides which evidence can be submitted when +// repair needs findings from each applicable validator. +func (r Request) AcceptsIndependentCandidate(validatorName string, modelDriven bool, candidate proposals.EnrichedCorrectionProposal) bool { + hasSpan := candidate.OriginalText != "" + switch validatorName { + case "proposal_shape": + return true + case "no_effect": + return hasSpan + case "original_text_presence": + return hasSpan && candidate.TargetSegmentID > 0 + case "protected_terms", "non_empty_resulting_segment": + return r.Preview(candidate).HasResult() + default: + return !modelDriven || r.Preview(candidate).Applicable + } +} diff --git a/internal/modules/glossary/module.go b/internal/modules/glossary/module.go index 6562843..3dfbab0 100644 --- a/internal/modules/glossary/module.go +++ b/internal/modules/glossary/module.go @@ -2,6 +2,7 @@ package glossary import ( "context" + frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation" @@ -11,7 +12,7 @@ import ( ) type Module struct { - validators []contracts.Validator + validators []frameworkvalidators.Validator } func New() (*Module, error) { @@ -33,8 +34,8 @@ func (m *Module) ReplacementPolicy() proposals.ReplacementPolicy { return proposals.ReplacementPolicyReplaceAll } -func (m *Module) Validators() []contracts.Validator { - return append([]contracts.Validator(nil), m.validators...) +func (m *Module) Validators() []frameworkvalidators.Validator { + return append([]frameworkvalidators.Validator(nil), m.validators...) } func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) { diff --git a/internal/modules/grammar/module.go b/internal/modules/grammar/module.go index 485aa3a..e174229 100644 --- a/internal/modules/grammar/module.go +++ b/internal/modules/grammar/module.go @@ -2,6 +2,7 @@ package grammar import ( "context" + frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation" @@ -11,7 +12,7 @@ import ( ) type Module struct { - validators []contracts.Validator + validators []frameworkvalidators.Validator } func New() (*Module, error) { @@ -33,8 +34,8 @@ func (m *Module) ReplacementPolicy() proposals.ReplacementPolicy { return proposals.ReplacementPolicyRequireUnique } -func (m *Module) Validators() []contracts.Validator { - return append([]contracts.Validator(nil), m.validators...) +func (m *Module) Validators() []frameworkvalidators.Validator { + return append([]frameworkvalidators.Validator(nil), m.validators...) } func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) { diff --git a/internal/modules/homophones/module.go b/internal/modules/homophones/module.go index f840e5a..f1a21e3 100644 --- a/internal/modules/homophones/module.go +++ b/internal/modules/homophones/module.go @@ -2,6 +2,7 @@ package homophones import ( "context" + frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation" @@ -11,7 +12,7 @@ import ( ) type Module struct { - validators []contracts.Validator + validators []frameworkvalidators.Validator } func New() (*Module, error) { @@ -33,8 +34,8 @@ func (m *Module) ReplacementPolicy() proposals.ReplacementPolicy { return proposals.ReplacementPolicyRequireUnique } -func (m *Module) Validators() []contracts.Validator { - return append([]contracts.Validator(nil), m.validators...) +func (m *Module) Validators() []frameworkvalidators.Validator { + return append([]frameworkvalidators.Validator(nil), m.validators...) } func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) { diff --git a/internal/modules/homophones/module_test.go b/internal/modules/homophones/module_test.go index 1706a9d..6123323 100644 --- a/internal/modules/homophones/module_test.go +++ b/internal/modules/homophones/module_test.go @@ -154,7 +154,7 @@ func TestHomophonesProtectedGlossaryTermBehaviorRejectsUnsafeCorrection(t *testi if err != nil { t.Fatalf("New error: %v", err) } - var protectedValidator contracts.Validator + var protectedValidator validators.Validator for _, v := range module.Validators() { if v.Name() == "protected_terms" { protectedValidator = v diff --git a/internal/modules/spoken_word/module.go b/internal/modules/spoken_word/module.go index 11898f4..6f97cea 100644 --- a/internal/modules/spoken_word/module.go +++ b/internal/modules/spoken_word/module.go @@ -2,6 +2,7 @@ package spoken_word import ( "context" + frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation" @@ -11,7 +12,7 @@ import ( ) type Module struct { - validators []contracts.Validator + validators []frameworkvalidators.Validator } func New() (*Module, error) { @@ -33,8 +34,8 @@ func (m *Module) ReplacementPolicy() proposals.ReplacementPolicy { return proposals.ReplacementPolicyRequireUnique } -func (m *Module) Validators() []contracts.Validator { - return append([]contracts.Validator(nil), m.validators...) +func (m *Module) Validators() []frameworkvalidators.Validator { + return append([]frameworkvalidators.Validator(nil), m.validators...) } func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (contracts.ProposalResult, error) { diff --git a/internal/modules/spoken_word/module_test.go b/internal/modules/spoken_word/module_test.go index 0e40767..47b400d 100644 --- a/internal/modules/spoken_word/module_test.go +++ b/internal/modules/spoken_word/module_test.go @@ -154,7 +154,7 @@ func TestSpokenWordProtectedGlossaryTermBehaviorRejectsUnsafeCorrection(t *testi if err != nil { t.Fatalf("New error: %v", err) } - var protectedValidator contracts.Validator + var protectedValidator validators.Validator for _, v := range module.Validators() { if v.Name() == "protected_terms" { protectedValidator = v diff --git a/internal/validators/chains.go b/internal/validators/chains.go index 3a96f21..ba0585e 100644 --- a/internal/validators/chains.go +++ b/internal/validators/chains.go @@ -2,10 +2,10 @@ package validators import ( "fmt" + frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators" "strings" "gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog" - "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" ) var builtInChains = map[string][]string{ @@ -58,7 +58,7 @@ func BuiltInChainKeys(moduleKey string) ([]string, error) { return out, nil } -func ResolveBuiltInChain(moduleKey string, registry *Registry) ([]contracts.Validator, error) { +func ResolveBuiltInChain(moduleKey string, registry *Registry) ([]frameworkvalidators.Validator, error) { moduleKey = strings.TrimSpace(moduleKey) keys, err := BuiltInChainKeys(moduleKey) if err != nil { @@ -68,7 +68,7 @@ func ResolveBuiltInChain(moduleKey string, registry *Registry) ([]contracts.Vali registry = NewBuiltInRegistry() } - out := make([]contracts.Validator, 0, len(keys)) + out := make([]frameworkvalidators.Validator, 0, len(keys)) for _, key := range keys { v, buildErr := registry.MustBuild(key) if buildErr != nil { diff --git a/internal/validators/editorial_review/validator.go b/internal/validators/editorial_review/validator.go index d28d336..b0bedb5 100644 --- a/internal/validators/editorial_review/validator.go +++ b/internal/validators/editorial_review/validator.go @@ -1,14 +1,13 @@ package editorial_review import ( - "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators" validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata" ) const Key = "editorial_review" -func New() (contracts.Validator, error) { +func New() (frameworkvalidators.Validator, error) { v, err := frameworkvalidators.NewLLMBackedValidator(Key, frameworkvalidators.LLMValidatorTypeEditorialReview) if err != nil { return nil, err diff --git a/internal/validators/meaning_reversal_review/validator.go b/internal/validators/meaning_reversal_review/validator.go index 156f3bf..5ac39aa 100644 --- a/internal/validators/meaning_reversal_review/validator.go +++ b/internal/validators/meaning_reversal_review/validator.go @@ -1,14 +1,13 @@ package meaning_reversal_review import ( - "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators" validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata" ) const Key = "meaning_reversal_review" -func New() (contracts.Validator, error) { +func New() (frameworkvalidators.Validator, error) { v, err := frameworkvalidators.NewLLMBackedValidator(Key, frameworkvalidators.LLMValidatorTypeMeaningReversal) if err != nil { return nil, err diff --git a/internal/validators/metadata/metadata.go b/internal/validators/metadata/metadata.go index 6179f3d..111301c 100644 --- a/internal/validators/metadata/metadata.go +++ b/internal/validators/metadata/metadata.go @@ -3,7 +3,7 @@ package metadata import ( "strings" - "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" + frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators" ) type ExecutionClass string @@ -37,11 +37,11 @@ var executionClassByKey = map[string]ExecutionClass{ } type ClassifiedValidator interface { - contracts.Validator + frameworkvalidators.Validator ExecutionClass() ExecutionClass } -func ClassOf(v contracts.Validator) ExecutionClass { +func ClassOf(v frameworkvalidators.Validator) ExecutionClass { if v == nil { return ExecutionClassDeterministic } @@ -69,7 +69,7 @@ func ClassForKey(key string) ExecutionClass { return class } -func Wrap(v contracts.Validator, class ExecutionClass) contracts.Validator { +func Wrap(v frameworkvalidators.Validator, class ExecutionClass) frameworkvalidators.Validator { if v == nil { return nil } @@ -83,7 +83,7 @@ func Wrap(v contracts.Validator, class ExecutionClass) contracts.Validator { } type wrappedValidator struct { - contracts.Validator + frameworkvalidators.Validator class ExecutionClass } diff --git a/internal/validators/metadata/metadata_test.go b/internal/validators/metadata/metadata_test.go index 3886d35..700394a 100644 --- a/internal/validators/metadata/metadata_test.go +++ b/internal/validators/metadata/metadata_test.go @@ -4,7 +4,6 @@ import ( "context" "testing" - "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators" ) @@ -12,7 +11,7 @@ type unclassifiedValidator struct{} func (u unclassifiedValidator) Name() string { return "unclassified" } -func (u unclassifiedValidator) Validate(_ context.Context, _ contracts.ValidationRequest) (frameworkvalidators.Result, error) { +func (u unclassifiedValidator) Validate(_ context.Context, _ frameworkvalidators.Request) (frameworkvalidators.Result, error) { return frameworkvalidators.Result{ValidatorName: u.Name(), Decisions: nil}, nil } @@ -22,7 +21,7 @@ type namedUnclassifiedValidator struct { func (n namedUnclassifiedValidator) Name() string { return n.name } -func (n namedUnclassifiedValidator) Validate(_ context.Context, _ contracts.ValidationRequest) (frameworkvalidators.Result, error) { +func (n namedUnclassifiedValidator) Validate(_ context.Context, _ frameworkvalidators.Request) (frameworkvalidators.Result, error) { return frameworkvalidators.Result{ValidatorName: n.Name(), Decisions: nil}, nil } diff --git a/internal/validators/module_chains_test.go b/internal/validators/module_chains_test.go index b0ea24a..b2928b0 100644 --- a/internal/validators/module_chains_test.go +++ b/internal/validators/module_chains_test.go @@ -3,7 +3,7 @@ package validators_test import ( "testing" - "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" + frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators" "gitea.maximumdirect.net/eric/audita/v3/internal/modules/glossary" "gitea.maximumdirect.net/eric/audita/v3/internal/modules/grammar" "gitea.maximumdirect.net/eric/audita/v3/internal/modules/homophones" @@ -20,30 +20,30 @@ func TestProductionModulesUseRegisteredValidatorKeys(t *testing.T) { cases := []struct { name string - build func() ([]contracts.Validator, error) + build func() ([]frameworkvalidators.Validator, error) }{ - {name: "glossary", build: func() ([]contracts.Validator, error) { + {name: "glossary", build: func() ([]frameworkvalidators.Validator, error) { m, err := glossary.New() if err != nil { return nil, err } return m.Validators(), nil }}, - {name: "homophones", build: func() ([]contracts.Validator, error) { + {name: "homophones", build: func() ([]frameworkvalidators.Validator, error) { m, err := homophones.New() if err != nil { return nil, err } return m.Validators(), nil }}, - {name: "spoken_word", build: func() ([]contracts.Validator, error) { + {name: "spoken_word", build: func() ([]frameworkvalidators.Validator, error) { m, err := spoken_word.New() if err != nil { return nil, err } return m.Validators(), nil }}, - {name: "grammar", build: func() ([]contracts.Validator, error) { + {name: "grammar", build: func() ([]frameworkvalidators.Validator, error) { m, err := grammar.New() if err != nil { return nil, err diff --git a/internal/validators/no_effect/validator.go b/internal/validators/no_effect/validator.go index b5711f2..7e67b00 100644 --- a/internal/validators/no_effect/validator.go +++ b/internal/validators/no_effect/validator.go @@ -1,13 +1,12 @@ package no_effect import ( - "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators" validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata" ) const Key = "no_effect" -func New() (contracts.Validator, error) { +func New() (frameworkvalidators.Validator, error) { return validatormetadata.Wrap(frameworkvalidators.NoEffectValidator{}, validatormetadata.ExecutionClassDeterministic), nil } diff --git a/internal/validators/non_empty_resulting_segment/validator.go b/internal/validators/non_empty_resulting_segment/validator.go index 4ef96a5..f90ccbe 100644 --- a/internal/validators/non_empty_resulting_segment/validator.go +++ b/internal/validators/non_empty_resulting_segment/validator.go @@ -1,13 +1,12 @@ package non_empty_resulting_segment import ( - "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators" validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata" ) const Key = "non_empty_resulting_segment" -func New() (contracts.Validator, error) { +func New() (frameworkvalidators.Validator, error) { return validatormetadata.Wrap(frameworkvalidators.NonEmptyResultingSegmentValidator{}, validatormetadata.ExecutionClassDeterministic), nil } diff --git a/internal/validators/original_text_presence/validator.go b/internal/validators/original_text_presence/validator.go index b139709..7d653d8 100644 --- a/internal/validators/original_text_presence/validator.go +++ b/internal/validators/original_text_presence/validator.go @@ -1,13 +1,12 @@ package original_text_presence import ( - "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators" validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata" ) const Key = "original_text_presence" -func New() (contracts.Validator, error) { +func New() (frameworkvalidators.Validator, error) { return validatormetadata.Wrap(frameworkvalidators.OriginalTextPresenceValidator{}, validatormetadata.ExecutionClassDeterministic), nil } diff --git a/internal/validators/proposal_shape/validator.go b/internal/validators/proposal_shape/validator.go index 70c0727..93bfc9b 100644 --- a/internal/validators/proposal_shape/validator.go +++ b/internal/validators/proposal_shape/validator.go @@ -1,13 +1,12 @@ package proposal_shape import ( - "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators" validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata" ) const Key = "proposal_shape" -func New() (contracts.Validator, error) { +func New() (frameworkvalidators.Validator, error) { return validatormetadata.Wrap(frameworkvalidators.ProposalShapeValidator{}, validatormetadata.ExecutionClassDeterministic), nil } diff --git a/internal/validators/protected_terms/validator.go b/internal/validators/protected_terms/validator.go index e8dcd05..83b7aec 100644 --- a/internal/validators/protected_terms/validator.go +++ b/internal/validators/protected_terms/validator.go @@ -3,14 +3,13 @@ package protected_terms import ( "context" - "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators" validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata" ) const Key = "protected_terms" -func New() (contracts.Validator, error) { +func New() (frameworkvalidators.Validator, error) { return validatormetadata.Wrap(validator{}, validatormetadata.ExecutionClassDeterministic), nil } @@ -18,6 +17,6 @@ type validator struct{} func (v validator) Name() string { return Key } -func (v validator) Validate(_ context.Context, req contracts.ValidationRequest) (frameworkvalidators.Result, error) { +func (v validator) Validate(_ context.Context, req frameworkvalidators.Request) (frameworkvalidators.Result, error) { return frameworkvalidators.ValidateProtectedTerms(req) } diff --git a/internal/validators/protected_terms/validator_test.go b/internal/validators/protected_terms/validator_test.go index 9b128ce..dc98f40 100644 --- a/internal/validators/protected_terms/validator_test.go +++ b/internal/validators/protected_terms/validator_test.go @@ -6,7 +6,6 @@ import ( "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" - "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators" validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata" @@ -50,7 +49,7 @@ func TestNewUsesGeneralBehaviorForGlossaryModule(t *testing.T) { t.Fatalf("New: %v", err) } - req := contracts.ValidationRequest{ + req := frameworkvalidators.Request{ ModuleKey: "glossary", Glossary: tinyGlossary(), CandidateProposal: []proposals.EnrichedCorrectionProposal{ @@ -78,7 +77,7 @@ func TestNewRejectsProtectedTermChangeOutsideGlossaryStage(t *testing.T) { t.Fatalf("New: %v", err) } - req := contracts.ValidationRequest{ + req := frameworkvalidators.Request{ WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Jesters"}}}, ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, ModuleKey: "homophones", @@ -115,7 +114,7 @@ func TestNewGlossaryStageRejectsProtectedTermChange(t *testing.T) { t.Fatalf("expected deterministic class, got %q", got) } - req := contracts.ValidationRequest{ + req := frameworkvalidators.Request{ WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Jesters"}}}, ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, ModuleKey: "glossary", @@ -146,7 +145,7 @@ func TestNewGlossaryStageAllowsProtectedTermInsertion(t *testing.T) { t.Fatalf("New: %v", err) } - req := contracts.ValidationRequest{ + req := frameworkvalidators.Request{ WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "gestures"}}}, ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, ModuleKey: "glossary", diff --git a/internal/validators/registry.go b/internal/validators/registry.go index b091c29..66b9284 100644 --- a/internal/validators/registry.go +++ b/internal/validators/registry.go @@ -2,9 +2,9 @@ package validators import ( "fmt" + frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators" "strings" - "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/v3/internal/validators/editorial_review" "gitea.maximumdirect.net/eric/audita/v3/internal/validators/meaning_reversal_review" validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata" @@ -30,7 +30,7 @@ const ( type BuiltInValidatorDefinition struct { Key string - Build func() (contracts.Validator, error) + Build func() (frameworkvalidators.Validator, error) } type Registry struct { @@ -64,7 +64,7 @@ func (r *Registry) Lookup(key string) (BuiltInValidatorDefinition, bool) { return def, ok } -func (r *Registry) MustBuild(key string) (contracts.Validator, error) { +func (r *Registry) MustBuild(key string) (frameworkvalidators.Validator, error) { if r == nil { return nil, fmt.Errorf("validator registry is nil") } diff --git a/internal/validators/registry_test.go b/internal/validators/registry_test.go index 4b01a68..795332e 100644 --- a/internal/validators/registry_test.go +++ b/internal/validators/registry_test.go @@ -2,12 +2,12 @@ package validators import ( "context" + frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators" "testing" "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary" "gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog" "gitea.maximumdirect.net/eric/audita/v3/internal/core/schema" - "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/v3/internal/validators/editorial_review" "gitea.maximumdirect.net/eric/audita/v3/internal/validators/meaning_reversal_review" @@ -50,7 +50,7 @@ func TestBuiltInValidatorPackagesConstruct(t *testing.T) { type validatorCtor struct { name string key string - build func() (contracts.Validator, error) + build func() (frameworkvalidators.Validator, error) wantClass validatormetadata.ExecutionClass } cases := []validatorCtor{ @@ -117,7 +117,7 @@ func TestRegistryProtectedTermsUsesNonGlossaryStageBehavior(t *testing.T) { t.Fatalf("must build protected_terms: %v", err) } - req := contracts.ValidationRequest{ + req := frameworkvalidators.Request{ ModuleKey: "glossary", Glossary: []glossary.Entry{{Collection: "party", ID: "jesters", Name: "Jesters", Category: "faction", Description: "Protected"}}, CandidateProposal: []proposals.EnrichedCorrectionProposal{ @@ -152,7 +152,7 @@ func TestResolveBuiltInChainUsesGlossaryStageProtectedTermsBehavior(t *testing.T t.Fatalf("resolve glossary chain: %v", err) } - var v contracts.Validator + var v frameworkvalidators.Validator for _, candidate := range chain { if candidate.Name() == KeyProtectedTerms { v = candidate @@ -163,7 +163,7 @@ func TestResolveBuiltInChainUsesGlossaryStageProtectedTermsBehavior(t *testing.T t.Fatalf("expected protected_terms validator in glossary chain") } - req := contracts.ValidationRequest{ + req := frameworkvalidators.Request{ WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Jesters"}}}, ReplacementPolicy: proposals.ReplacementPolicyReplaceAll, ModuleKey: "glossary", diff --git a/internal/validators/spoken_form_plausibility/validator.go b/internal/validators/spoken_form_plausibility/validator.go index eaed941..1d4b974 100644 --- a/internal/validators/spoken_form_plausibility/validator.go +++ b/internal/validators/spoken_form_plausibility/validator.go @@ -1,14 +1,13 @@ package spoken_form_plausibility import ( - "gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts" frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators" validatormetadata "gitea.maximumdirect.net/eric/audita/v3/internal/validators/metadata" ) const Key = "spoken_form_plausibility" -func New() (contracts.Validator, error) { +func New() (frameworkvalidators.Validator, error) { v, err := frameworkvalidators.NewLLMBackedValidator(Key, frameworkvalidators.LLMValidatorTypeSpokenFormPlausibility) if err != nil { return nil, err