audita/python/tests/test_llm_validators.py

import threading

import pytest

from audita.core.chunking import TokenBatch
from audita.core.config import AuditaConfig, ConfigOverrides
from audita.core.errors import AuditaLLMError
from audita.core.schemas import parse_glossary_yaml, parse_transcript_json
from audita.framework.models import CorrectionProposal, ModuleRunSpec
from audita.validators import (
    EditorialValidator,
    GrammarOnlyValidator,
    IdenticalTextValidator,
    NonEmptySegmentValidator,
    OriginalTextPresentValidator,
)
from audita.validators.base import ValidationContext
from audita.validators.llm import MeaningReversalValidator, SpokenFormPlausibilityValidator, SpokenWordValidator
from audita.validators.prompts import (
    build_grammar_only_messages,
    build_meaning_reversal_messages,
    build_spoken_form_plausibility_messages,
    build_spoken_word_messages,
)
import audita.validators.llm as llm_module


class _Module:
    def __init__(self, replacement_policy: str) -> None:
        self.replacement_policy = replacement_policy


class FakeStructuredLLMClient:
    def __init__(self, responses):
        self._responses = list(responses)
        self.calls = []

    def run_structured(self, *, stage_name, messages, response_model, config):
        self.calls.append(
            {
                "stage_name": stage_name,
                "messages": list(messages),
                "response_model": response_model,
                "config": config,
            }
        )
        if not self._responses:
            raise AuditaLLMError("FakeStructuredLLMClient received more calls than expected.")
        return response_model.model_validate(self._responses.pop(0))


class CoordinatedStructuredLLMClient:
    def __init__(self, responses, barrier):
        self._responses = list(responses)
        self._barrier = barrier
        self._lock = threading.Lock()
        self.calls = []

    def run_structured(self, *, stage_name, messages, response_model, config):
        self._barrier.wait()
        with self._lock:
            self.calls.append(
                {
                    "stage_name": stage_name,
                    "messages": list(messages),
                    "response_model": response_model,
                    "config": config,
                }
            )
            if not self._responses:
                raise AuditaLLMError("CoordinatedStructuredLLMClient received more calls than expected.")
            payload = self._responses.pop(0)
            if callable(payload):
                payload = payload(stage_name=stage_name, messages=messages, response_model=response_model, config=config)
        return response_model.model_validate(payload)


def _glossary():
    return parse_glossary_yaml(
        """
        glossary:
          - name: "Jesters"
            category: faction
            summary: "The Jesters are a faction."
          - name: "Hrank"
            category: pc
            summary: "Hrank is a player character."
          - name: "Lyra"
            category: npc
            summary: "Lyra is an NPC."
        """
    )


def _context(
    *,
    proposals,
    transcript,
    llm_client,
    tmp_path,
    replacement_policy="require_unique",
    llm_concurrency=1,
    validation_llm_concurrency=None,
):
    config = AuditaConfig.from_sources(
        env={"OPENROUTER_API_KEY": "test-key"},
        overrides=ConfigOverrides(
            llm_concurrency=llm_concurrency,
            validation_llm_concurrency=validation_llm_concurrency,
        ),
    )
    return ValidationContext(
        proposals=proposals,
        transcript=transcript,
        glossary=_glossary(),
        config=config.validation_llm_config(),
        run_spec=ModuleRunSpec(
            instance_name="homophones",
            module_key="homophones",
            module=_Module(replacement_policy),
        ),
        run_dir=tmp_path,
        llm_client=llm_client,
    )


def test_spoken_form_plausibility_validator_approves_plausible_and_rejects_implausible_corrections(tmp_path):
    transcript = parse_transcript_json(
        """
        [
          {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple.", "categories": ["narration"]},
          {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "Lyra moved first.", "categories": ["combat"]}
        ]
        """
    )
    proposals = [
        CorrectionProposal(
            proposal_index=0,
            module_instance="homophones",
            module_key="homophones",
            id=1,
            original_text="gestures",
            corrected_text="Jesters",
            confidence=0.95,
        ),
        CorrectionProposal(
            proposal_index=1,
            module_instance="homophones",
            module_key="homophones",
            id=2,
            original_text="Lyra",
            corrected_text="Jesters",
            confidence=0.95,
        ),
    ]
    client = FakeStructuredLLMClient(
        [
            {
                "validations": [
                    {
                        "correction_index": 0,
                        "approved": True,
                        "confidence": 0.97,
                        "reason": "Likely spoken-form correction in context.",
                    },
                    {
                        "correction_index": 1,
                        "approved": False,
                        "confidence": 0.99,
                        "reason": "Not plausibly related by homophone or mistranscription.",
                    },
                ]
            }
        ]
    )

    result = SpokenFormPlausibilityValidator("spoken_form_plausibility_review").validate(
        _context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)
    )

    assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [
        (0, True),
        (1, False),
    ]
    prompt_text = client.calls[0]["messages"][1]["content"]
    assert '"original_segment_text"' in prompt_text
    assert '"categories"' in prompt_text
    assert "There were Jesters at the temple." in prompt_text
    assert "Lyra moved first." in prompt_text


def test_meaning_reversal_validator_rejects_reversal_and_approves_nonreversal(tmp_path):
    transcript = parse_transcript_json(
        """
        [
          {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "ChatGPT still can't really do that with a dam, but Claude actually can."},
          {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "The figure became visible in the doorway."}
        ]
        """
    )
    proposals = [
        CorrectionProposal(
            proposal_index=0,
            module_instance="homophones",
            module_key="homophones",
            id=1,
            original_text="dam",
            corrected_text="damn",
            confidence=0.95,
        ),
        CorrectionProposal(
            proposal_index=1,
            module_instance="homophones",
            module_key="homophones",
            id=2,
            original_text="visible",
            corrected_text="invisible",
            confidence=0.95,
        ),
    ]
    client = FakeStructuredLLMClient(
        [
            {
                "validations": [
                    {
                        "correction_index": 0,
                        "approved": True,
                        "confidence": 0.94,
                        "reason": "Does not reverse the segment meaning.",
                    },
                    {
                        "correction_index": 1,
                        "approved": False,
                        "confidence": 1.0,
                        "reason": "Changes visible to invisible and reverses the segment meaning.",
                    },
                ]
            }
        ]
    )

    result = MeaningReversalValidator("meaning_reversal_review").validate(
        _context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)
    )

    assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [
        (0, True),
        (1, False),
    ]
    prompt_text = client.calls[0]["messages"][1]["content"]
    assert "The figure became visible in the doorway." in prompt_text
    assert "The figure became invisible in the doorway." in prompt_text


def test_spoken_word_validator_approves_cleanup_and_rejects_rewrite(tmp_path):
    transcript = parse_transcript_json(
        """
        [
          {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "I, uh, I think we should go."},
          {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "We should maybe proceed carefully."}
        ]
        """
    )
    proposals = [
        CorrectionProposal(
            proposal_index=0,
            module_instance="spoken_word",
            module_key="spoken_word",
            id=1,
            original_text="I, uh, I think",
            corrected_text="I think",
            confidence=0.95,
        ),
        CorrectionProposal(
            proposal_index=1,
            module_instance="spoken_word",
            module_key="spoken_word",
            id=2,
            original_text="maybe proceed carefully",
            corrected_text="go now",
            confidence=0.95,
        ),
    ]
    client = FakeStructuredLLMClient(
        [
            {
                "validations": [
                    {
                        "correction_index": 0,
                        "approved": True,
                        "confidence": 0.97,
                        "reason": "Reasonable dysfluency cleanup that preserves meaning.",
                    },
                    {
                        "correction_index": 1,
                        "approved": False,
                        "confidence": 0.99,
                        "reason": "This changes the substance of the segment rather than cleaning a dysfluency.",
                    },
                ]
            }
        ]
    )

    result = SpokenWordValidator("spoken_word_review").validate(
        _context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)
    )

    assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [
        (0, True),
        (1, False),
    ]
    prompt_text = client.calls[0]["messages"][1]["content"]
    assert "I think we should go." in prompt_text
    assert "go now" in prompt_text
    assert "acceptable editorial revision" in prompt_text
    assert "homophone or mistranscription corrections" in prompt_text


def test_spoken_word_validator_allows_punctuation_cleanup_tied_to_dysfluency(tmp_path):
    transcript = parse_transcript_json(
        """
        [
          {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "Well ... I think we should go."}
        ]
        """
    )
    proposals = [
        CorrectionProposal(
            proposal_index=0,
            module_instance="spoken_word",
            module_key="spoken_word",
            id=1,
            original_text="Well ... ",
            corrected_text="",
            confidence=0.95,
        )
    ]
    client = FakeStructuredLLMClient(
        [
            {
                "validations": [
                    {
                        "correction_index": 0,
                        "approved": True,
                        "confidence": 0.95,
                        "reason": "Removes a hesitation artifact without changing substantive meaning.",
                    }
                ]
            }
        ]
    )

    result = SpokenWordValidator("spoken_word_review").validate(
        _context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)
    )

    assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [(0, True)]


def test_grammar_only_validator_approves_editorial_cleanup(tmp_path):
    transcript = parse_transcript_json(
        """
        [
          {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "hello there"},
          {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "cant we go"},
          {"id": 3, "speaker": "A", "start": 2.0, "end": 3.0, "text": "Hello,world"}
        ]
        """
    )
    proposals = [
        CorrectionProposal(
            proposal_index=0,
            module_instance="grammar",
            module_key="grammar",
            id=1,
            original_text="hello there",
            corrected_text="Hello there.",
            confidence=0.95,
        ),
        CorrectionProposal(
            proposal_index=1,
            module_instance="grammar",
            module_key="grammar",
            id=2,
            original_text="cant",
            corrected_text="can't",
            confidence=0.95,
        ),
        CorrectionProposal(
            proposal_index=2,
            module_instance="grammar",
            module_key="grammar",
            id=3,
            original_text="Hello,world",
            corrected_text="Hello, world",
            confidence=0.95,
        ),
    ]
    client = FakeStructuredLLMClient(
        [
            {
                "validations": [
                    {
                        "correction_index": 0,
                        "approved": True,
                        "confidence": 0.98,
                        "reason": "Conservative punctuation and capitalization cleanup.",
                    },
                    {
                        "correction_index": 1,
                        "approved": True,
                        "confidence": 0.95,
                        "reason": "Conservative apostrophe insertion within grammar cleanup.",
                    },
                    {
                        "correction_index": 2,
                        "approved": True,
                        "confidence": 0.97,
                        "reason": "Conservative spacing cleanup.",
                    },
                ]
            }
        ]
    )

    result = GrammarOnlyValidator("grammar_only_guard").validate(
        _context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)
    )

    assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [
        (0, True),
        (1, True),
        (2, True),
    ]
    prompt_text = client.calls[0]["messages"][1]["content"]
    assert "acceptable editorial revision" in prompt_text
    assert "article cleanup" in prompt_text
    assert "dysfluency cleanup" in prompt_text


def test_grammar_only_validator_allows_indefinite_article_changes(tmp_path):
    transcript = parse_transcript_json(
        """
        [
          {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "a intelligence saving throw"},
          {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "an owl"}
        ]
        """
    )
    proposals = [
        CorrectionProposal(
            proposal_index=0,
            module_instance="grammar",
            module_key="grammar",
            id=1,
            original_text="a intelligence saving throw",
            corrected_text="an intelligence saving throw",
            confidence=0.95,
        ),
        CorrectionProposal(
            proposal_index=1,
            module_instance="grammar",
            module_key="grammar",
            id=2,
            original_text="an owl",
            corrected_text="a owl",
            confidence=0.95,
        ),
    ]
    client = FakeStructuredLLMClient(
        [
            {
                "validations": [
                    {
                        "correction_index": 0,
                        "approved": True,
                        "confidence": 0.96,
                        "reason": "Allowed whole-word article cleanup.",
                    },
                    {
                        "correction_index": 1,
                        "approved": True,
                        "confidence": 0.93,
                        "reason": "Allowed whole-word article cleanup.",
                    },
                ]
            }
        ]
    )

    result = GrammarOnlyValidator("grammar_only_guard").validate(
        _context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)
    )

    assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [
        (0, True),
        (1, True),
    ]


def test_grammar_only_validator_allows_editorial_overlap_and_rejects_meaning_change(tmp_path):
    transcript = parse_transcript_json(
        """
        [
          {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "their plan"},
          {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "dam"},
          {"id": 3, "speaker": "A", "start": 2.0, "end": 3.0, "text": "uh"},
          {"id": 4, "speaker": "A", "start": 3.0, "end": 4.0, "text": "I I agree"},
          {"id": 5, "speaker": "A", "start": 4.0, "end": 5.0, "text": "Eric"}
        ]
        """
    )
    proposals = [
        CorrectionProposal(
            proposal_index=0,
            module_instance="grammar",
            module_key="grammar",
            id=1,
            original_text="their",
            corrected_text="there",
            confidence=0.95,
        ),
        CorrectionProposal(
            proposal_index=1,
            module_instance="grammar",
            module_key="grammar",
            id=2,
            original_text="dam",
            corrected_text="damn",
            confidence=0.95,
        ),
        CorrectionProposal(
            proposal_index=2,
            module_instance="grammar",
            module_key="grammar",
            id=3,
            original_text="uh",
            corrected_text="",
            confidence=0.95,
        ),
        CorrectionProposal(
            proposal_index=3,
            module_instance="grammar",
            module_key="grammar",
            id=4,
            original_text="I I",
            corrected_text="I",
            confidence=0.95,
        ),
        CorrectionProposal(
            proposal_index=4,
            module_instance="grammar",
            module_key="grammar",
            id=5,
            original_text="Eric",
            corrected_text="Eric's",
            confidence=0.95,
        ),
    ]
    client = FakeStructuredLLMClient(
        [
            {
                "validations": [
                    {
                        "correction_index": 0,
                        "approved": False,
                        "confidence": 0.99,
                        "reason": "This changes meaning rather than preserving it through editorial cleanup.",
                    },
                    {
                        "correction_index": 1,
                        "approved": True,
                        "confidence": 0.93,
                        "reason": "Low-risk editorial homophone correction that preserves meaning.",
                    },
                    {
                        "correction_index": 2,
                        "approved": True,
                        "confidence": 0.95,
                        "reason": "Editorial dysfluency cleanup that preserves meaning.",
                    },
                    {
                        "correction_index": 3,
                        "approved": True,
                        "confidence": 0.95,
                        "reason": "Editorial repetition cleanup that preserves meaning.",
                    },
                    {
                        "correction_index": 4,
                        "approved": True,
                        "confidence": 0.91,
                        "reason": "Conservative editorial possessive correction that preserves meaning.",
                    },
                ]
            }
        ]
    )

    result = GrammarOnlyValidator("grammar_only_guard").validate(
        _context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)
    )

    assert [decision.approved for decision in result.decisions] == [False, True, True, True, True]


def test_grammar_only_validator_allows_embedded_homophone_fix_within_grammar_revision(tmp_path):
    transcript = parse_transcript_json(
        """
        [
          {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "question. if he dies does he stay there the way that yeah the way that it's written it's like so if it stops that he goes down but what if it doesn't"}
        ]
        """
    )
    proposals = [
        CorrectionProposal(
            proposal_index=0,
            module_instance="grammar",
            module_key="grammar",
            id=1,
            original_text="question. if he dies does he stay there the way that yeah the way that it's written it's like so if it stops that he goes down but what if it doesn't",
            corrected_text="question: If he dies, does he stay there? The way that, yeah, the way that it's written, it's like, so if it stops, then he goes down; but what if it doesn't?",
            confidence=0.95,
        )
    ]
    client = FakeStructuredLLMClient(
        [
            {
                "validations": [
                    {
                        "correction_index": 0,
                        "approved": True,
                        "confidence": 0.94,
                        "reason": "Primarily a grammatical revision with an embedded likely mistranscription recovery.",
                    }
                ]
            }
        ]
    )

    result = GrammarOnlyValidator("grammar_only_guard").validate(
        _context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)
    )

    assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [(0, True)]


def test_non_empty_segment_validator_rejects_empty_and_whitespace_only_segments(tmp_path):
    transcript = parse_transcript_json(
        """
        [
          {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "uh"},
          {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "um"}
        ]
        """
    )
    proposals = [
        CorrectionProposal(
            proposal_index=0,
            module_instance="spoken_word",
            module_key="spoken_word",
            id=1,
            original_text="uh",
            corrected_text="",
            confidence=0.95,
        ),
        CorrectionProposal(
            proposal_index=1,
            module_instance="spoken_word",
            module_key="spoken_word",
            id=2,
            original_text="um",
            corrected_text="   ",
            confidence=0.95,
        ),
    ]

    result = NonEmptySegmentValidator("non_empty_segment_guard").validate(
        _context(proposals=proposals, transcript=transcript, llm_client=None, tmp_path=tmp_path)
    )

    assert [decision.approved for decision in result.decisions] == [False, False]
    assert all(decision.reason == "correction would leave the segment empty" for decision in result.decisions)


def test_non_empty_segment_validator_allows_punctuation_only_and_unpreviewable_proposals(tmp_path):
    transcript = parse_transcript_json(
        """
        [
          {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "uh"},
          {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "There were gestures at the temple."}
        ]
        """
    )
    proposals = [
        CorrectionProposal(
            proposal_index=0,
            module_instance="spoken_word",
            module_key="spoken_word",
            id=1,
            original_text="uh",
            corrected_text=".",
            confidence=0.95,
        ),
        CorrectionProposal(
            proposal_index=1,
            module_instance="spoken_word",
            module_key="spoken_word",
            id=2,
            original_text="rank",
            corrected_text="Hrank",
            confidence=0.95,
        ),
    ]

    result = NonEmptySegmentValidator("non_empty_segment_guard").validate(
        _context(proposals=proposals, transcript=transcript, llm_client=None, tmp_path=tmp_path)
    )

    assert [(decision.proposal_index, decision.approved, decision.reason) for decision in result.decisions] == [
        (0, True, None),
        (1, True, None),
    ]


def test_identical_text_validator_rejects_exact_noops_only(tmp_path):
    transcript = parse_transcript_json(
        """
        [
          {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "hello"},
          {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "hello"}
        ]
        """
    )
    proposals = [
        CorrectionProposal(
            proposal_index=0,
            module_instance="grammar",
            module_key="grammar",
            id=1,
            original_text="hello",
            corrected_text="hello",
            confidence=0.95,
        ),
        CorrectionProposal(
            proposal_index=1,
            module_instance="grammar",
            module_key="grammar",
            id=2,
            original_text="hello",
            corrected_text="Hello",
            confidence=0.95,
        ),
    ]

    result = IdenticalTextValidator("identical_text_guard").validate(
        _context(proposals=proposals, transcript=transcript, llm_client=None, tmp_path=tmp_path)
    )

    assert [(decision.proposal_index, decision.approved, decision.reason) for decision in result.decisions] == [
        (0, False, "proposal original_text and corrected_text are identical"),
        (1, True, None),
    ]


def test_original_text_present_validator_rejects_missing_spans_and_allows_present_or_unknown_segments(tmp_path):
    transcript = parse_transcript_json(
        """
        [
          {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "hello hello"},
          {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "goodbye"}
        ]
        """
    )
    proposals = [
        CorrectionProposal(
            proposal_index=0,
            module_instance="homophones",
            module_key="homophones",
            id=1,
            original_text="hello",
            corrected_text="hi",
            confidence=0.95,
        ),
        CorrectionProposal(
            proposal_index=1,
            module_instance="homophones",
            module_key="homophones",
            id=2,
            original_text="rank",
            corrected_text="Hrank",
            confidence=0.95,
        ),
        CorrectionProposal(
            proposal_index=2,
            module_instance="homophones",
            module_key="homophones",
            id=99,
            original_text="missing",
            corrected_text="present",
            confidence=0.95,
        ),
    ]

    result = OriginalTextPresentValidator("original_text_present_guard").validate(
        _context(proposals=proposals, transcript=transcript, llm_client=None, tmp_path=tmp_path)
    )

    assert [(decision.proposal_index, decision.approved, decision.reason) for decision in result.decisions] == [
        (0, True, None),
        (1, False, "proposal original_text does not match segment text"),
        (2, True, None),
    ]


@pytest.mark.parametrize(
    ("validator", "payload", "message_fragment"),
    [
        (
            SpokenFormPlausibilityValidator("spoken_form_plausibility_review"),
            {
                "validations": [
                    {
                        "correction_index": 0,
                        "approved": True,
                        "confidence": 0.9,
                        "reason": "ok",
                    },
                    {
                        "correction_index": 0,
                        "approved": True,
                        "confidence": 0.9,
                        "reason": "duplicate",
                    },
                ]
            },
            "duplicate correction_index values",
        ),
        (
            MeaningReversalValidator("meaning_reversal_review"),
            {"validations": []},
            "omitted correction_index values",
        ),
        (
            SpokenFormPlausibilityValidator("spoken_form_plausibility_review"),
            {
                "validations": [
                    {
                        "correction_index": 99,
                        "approved": True,
                        "confidence": 0.9,
                        "reason": "unknown",
                    }
                ]
            },
            "unknown correction_index",
        ),
        (
            EditorialValidator("editorial_review"),
            {
                "validations": [
                    {
                        "correction_index": 99,
                        "approved": True,
                        "confidence": 0.9,
                        "reason": "unknown",
                    }
                ]
            },
            "unknown correction_index",
        ),
        (
            SpokenWordValidator("spoken_word_review"),
            {
                "validations": [
                    {
                        "correction_index": 99,
                        "approved": True,
                        "confidence": 0.9,
                        "reason": "unknown",
                    }
                ]
            },
            "unknown correction_index",
        ),
    ],
)
def test_llm_validators_reject_bad_correction_indexes(tmp_path, validator, payload, message_fragment):
    transcript = parse_transcript_json(
        """
        [
          {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."}
        ]
        """
    )
    proposals = [
        CorrectionProposal(
            proposal_index=0,
            module_instance="homophones",
            module_key="homophones",
            id=1,
            original_text="gestures",
            corrected_text="Jesters",
            confidence=0.95,
        )
    ]
    client = FakeStructuredLLMClient([payload])

    with pytest.raises(AuditaLLMError, match=message_fragment):
        validator.validate(_context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path))


def test_llm_validator_rejects_unpreviewable_proposals_without_calling_llm(tmp_path):
    transcript = parse_transcript_json(
        """
        [
          {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."}
        ]
        """
    )
    proposals = [
        CorrectionProposal(
            proposal_index=0,
            module_instance="homophones",
            module_key="homophones",
            id=1,
            original_text="rank",
            corrected_text="Hrank",
            confidence=0.95,
        )
    ]
    client = FakeStructuredLLMClient([])

    result = SpokenFormPlausibilityValidator("spoken_form_plausibility_review").validate(
        _context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)
    )

    assert result.decisions[0].approved is False
    assert result.decisions[0].reason == "proposal original_text does not match segment text"
    assert client.calls == []


def test_llm_validator_skips_empty_candidate_sets_without_calling_llm(tmp_path):
    client = FakeStructuredLLMClient([])

    result = MeaningReversalValidator("meaning_reversal_review").validate(
        _context(proposals=[], transcript=[], llm_client=client, tmp_path=tmp_path)
    )

    assert result.decisions == []
    assert client.calls == []


def test_spoken_form_prompt_emphasizes_acoustic_plausibility():
    messages = build_spoken_form_plausibility_messages(
        [
            {
                "correction_index": 0,
                "id": 1,
                "original_segment_text": "There were gestures at the temple.",
                "corrected_segment_text": "There were Jesters at the temple.",
                "original_text": "gestures",
                "corrected_text": "Jesters",
            }
        ]
    )

    combined = messages[0]["content"] + messages[1]["content"]
    assert "homophone" in combined
    assert "gestures" in combined
    assert "Lyra" in combined
    assert '"original_segment_text"' in messages[1]["content"]


def test_meaning_reversal_prompt_emphasizes_antonyms_and_segment_context():
    messages = build_meaning_reversal_messages(
        [
            {
                "correction_index": 0,
                "id": 1,
                "original_segment_text": "The figure became visible in the doorway.",
                "corrected_segment_text": "The figure became invisible in the doorway.",
                "original_text": "visible",
                "corrected_text": "invisible",
            }
        ]
    )

    combined = messages[0]["content"] + messages[1]["content"]
    assert "antonym" in combined
    assert "visible" in combined
    assert "up" in combined
    assert "original_segment_text" in messages[1]["content"]


def test_spoken_word_prompt_emphasizes_dysfluency_cleanup():
    messages = build_spoken_word_messages(
        [
            {
                "correction_index": 0,
                "id": 1,
                "original_segment_text": "I, uh, I think we should go.",
                "corrected_segment_text": "I think we should go.",
                "original_text": "I, uh, I think",
                "corrected_text": "I think",
            }
        ]
    )

    combined = messages[0]["content"] + messages[1]["content"]
    assert "dysfluencies" in combined
    assert "intentional emphasis" in combined
    assert "Stop! Stop! Stop!" in combined
    assert "punctuation" in combined
    assert "substantive meaning" in combined
    assert "original_segment_text" in messages[1]["content"]


def test_llm_validators_use_shared_token_batching_helper(monkeypatch, tmp_path):
    transcript = parse_transcript_json(
        """
        [
          {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."},
          {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "We saw rank near the gate."},
          {"id": 3, "speaker": "A", "start": 2.0, "end": 3.0, "text": "ChatGPT still can't do that with a dam."}
        ]
        """
    )
    proposals = [
        CorrectionProposal(
            proposal_index=0,
            module_instance="homophones",
            module_key="homophones",
            id=1,
            original_text="gestures",
            corrected_text="Jesters",
            confidence=0.95,
        ),
        CorrectionProposal(
            proposal_index=1,
            module_instance="homophones",
            module_key="homophones",
            id=2,
            original_text="rank",
            corrected_text="Hrank",
            confidence=0.95,
        ),
        CorrectionProposal(
            proposal_index=2,
            module_instance="homophones",
            module_key="homophones",
            id=3,
            original_text="dam",
            corrected_text="damn",
            confidence=0.95,
        ),
    ]
    client = FakeStructuredLLMClient(
        [
            {
                "validations": [
                    {
                        "correction_index": 0,
                        "approved": True,
                        "confidence": 0.9,
                        "reason": "ok",
                    }
                ]
            },
            {
                "validations": [
                    {
                        "correction_index": 1,
                        "approved": True,
                        "confidence": 0.9,
                        "reason": "ok",
                    },
                    {
                        "correction_index": 2,
                        "approved": True,
                        "confidence": 0.9,
                        "reason": "ok",
                    },
                ]
            },
        ]
    )
    chunk_calls = []

    def fake_chunk_payload_items(items, max_tokens, payload_fn, empty_error_message):
        chunk_calls.append(
            {
                "max_tokens": max_tokens,
                "payloads": [payload_fn(item) for item in items],
                "empty_error_message": empty_error_message,
            }
        )
        return [
            TokenBatch(batch_index=0, items=list(items[:1]), token_count=1),
            TokenBatch(batch_index=1, items=list(items[1:]), token_count=1),
        ]

    monkeypatch.setattr(llm_module, "chunk_payload_items", fake_chunk_payload_items)

    result = SpokenFormPlausibilityValidator("spoken_form_plausibility_review").validate(
        _context(proposals=proposals, transcript=transcript, llm_client=client, tmp_path=tmp_path)
    )

    assert [decision.approved for decision in result.decisions] == [True, True, True]
    assert len(client.calls) == 2
    assert len(chunk_calls) == 1
    assert (
        chunk_calls[0]["max_tokens"]
        == AuditaConfig.from_sources(env={"OPENROUTER_API_KEY": "test-key"}).validation_max_prompt_tokens
    )
    assert all("corrected_segment_text" in payload for payload in chunk_calls[0]["payloads"])


def test_grammar_validation_prompt_is_scoped_to_editorial_cleanup():
    messages = build_grammar_only_messages(
        [
            {
                "correction_index": 0,
                "id": 1,
                "original_text": "dam",
                "corrected_text": "damn",
                "confidence": 0.95,
                "original_segment_text": "ChatGPT still can't do that with a dam.",
                "corrected_segment_text": "ChatGPT still can't do that with a damn.",
            }
        ]
    )

    combined = messages[0]["content"] + messages[1]["content"]
    assert "acceptable editorial revision" in combined
    assert "article cleanup" in combined
    assert "homophone or mistranscription corrections" in combined
    assert "dysfluency cleanup" in combined
    assert "original_segment_text" in messages[1]["content"]


def test_llm_validators_process_batches_concurrently_and_preserve_proposal_order(monkeypatch, tmp_path):
    transcript = parse_transcript_json(
        """
        [
          {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."},
          {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "ChatGPT still can't do that with a dam."}
        ]
        """
    )
    proposals = [
        CorrectionProposal(
            proposal_index=0,
            module_instance="homophones",
            module_key="homophones",
            id=1,
            original_text="gestures",
            corrected_text="Jesters",
            confidence=0.95,
        ),
        CorrectionProposal(
            proposal_index=1,
            module_instance="homophones",
            module_key="homophones",
            id=2,
            original_text="dam",
            corrected_text="damn",
            confidence=0.95,
        ),
    ]
    client = CoordinatedStructuredLLMClient(
        [
            lambda **kwargs: {
                "validations": [
                    {
                        "correction_index": 0 if '"correction_index": 0' in kwargs["messages"][1]["content"] else 1,
                        "approved": True,
                        "confidence": 0.97,
                        "reason": "ok",
                    }
                ]
            },
            lambda **kwargs: {
                "validations": [
                    {
                        "correction_index": 0 if '"correction_index": 0' in kwargs["messages"][1]["content"] else 1,
                        "approved": True,
                        "confidence": 0.94,
                        "reason": "ok",
                    }
                ]
            },
        ],
        threading.Barrier(2, timeout=1.0),
    )

    def fake_chunk_payload_items(items, max_tokens, payload_fn, empty_error_message):
        return [
            TokenBatch(batch_index=0, items=list(items[:1]), token_count=1),
            TokenBatch(batch_index=1, items=list(items[1:]), token_count=1),
        ]

    monkeypatch.setattr(llm_module, "chunk_payload_items", fake_chunk_payload_items)

    result = SpokenFormPlausibilityValidator("spoken_form_plausibility_review").validate(
        _context(
            proposals=proposals,
            transcript=transcript,
            llm_client=client,
            tmp_path=tmp_path,
            llm_concurrency=2,
        )
    )

    assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [
        (0, True),
        (1, True),
    ]
    assert len(client.calls) == 2


def test_validator_uses_validation_llm_config(tmp_path):
    transcript = parse_transcript_json(
        """
        [
          {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."}
        ]
        """
    )
    proposals = [
        CorrectionProposal(
            proposal_index=0,
            module_instance="homophones",
            module_key="homophones",
            id=1,
            original_text="gestures",
            corrected_text="Jesters",
            confidence=0.95,
        )
    ]
    client = FakeStructuredLLMClient(
        [
            {
                "validations": [
                    {
                        "correction_index": 0,
                        "approved": True,
                        "confidence": 0.97,
                        "reason": "ok",
                    }
                ]
            }
        ]
    )
    config = AuditaConfig.from_sources(
        env={"OPENROUTER_API_KEY": "primary-key"},
        overrides=ConfigOverrides(
            model="primary-model",
            base_url="http://localhost:8000/v1",
            max_retries=7,
            llm_timeout_seconds=120,
            validation_llm_api_key="validation-key",
            validation_model="validation-model",
            validation_base_url="http://localhost:9000/v1",
            validation_max_retries=2,
            validation_llm_timeout_seconds=240,
            validation_llm_concurrency=3,
        ),
    )
    context = ValidationContext(
        proposals=proposals,
        transcript=transcript,
        glossary=_glossary(),
        config=config.validation_llm_config(),
        run_spec=ModuleRunSpec(
            instance_name="homophones",
            module_key="homophones",
            module=_Module("require_unique"),
        ),
        run_dir=tmp_path,
        llm_client=client,
    )

    SpokenFormPlausibilityValidator("spoken_form_plausibility_review").validate(context)

    validation_config = client.calls[0]["config"]
    assert validation_config.api_key == "validation-key"
    assert validation_config.model == "validation-model"
    assert validation_config.base_url == "http://localhost:9000/v1"
    assert validation_config.max_retries == 2
    assert validation_config.llm_timeout_seconds == 240
    assert validation_config.llm_concurrency == 3


def test_validator_batches_use_validation_max_prompt_tokens(tmp_path, monkeypatch):
    transcript = parse_transcript_json(
        """
        [
          {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."}
        ]
        """
    )
    proposals = [
        CorrectionProposal(
            proposal_index=0,
            module_instance="homophones",
            module_key="homophones",
            id=1,
            original_text="gestures",
            corrected_text="Jesters",
            confidence=0.95,
        )
    ]
    client = FakeStructuredLLMClient(
        [
            {
                "validations": [
                    {
                        "correction_index": 0,
                        "approved": True,
                        "confidence": 0.97,
                        "reason": "ok",
                    }
                ]
            }
        ]
    )
    chunk_calls = []

    def fake_chunk_payload_items(items, max_tokens, payload_fn, empty_error_message):
        chunk_calls.append(
            {
                "count": len(items),
                "max_tokens": max_tokens,
            }
        )
        return [TokenBatch(batch_index=0, items=list(items), token_count=1)]

    monkeypatch.setattr(llm_module, "chunk_payload_items", fake_chunk_payload_items)
    config = AuditaConfig.from_sources(
        env={"OPENROUTER_API_KEY": "test-key"},
        overrides=ConfigOverrides(validation_max_prompt_tokens=1024),
    )
    context = ValidationContext(
        proposals=proposals,
        transcript=transcript,
        glossary=_glossary(),
        config=config.validation_llm_config(),
        run_spec=ModuleRunSpec(
            instance_name="homophones",
            module_key="homophones",
            module=_Module("require_unique"),
        ),
        run_dir=tmp_path,
        llm_client=client,
    )

    SpokenFormPlausibilityValidator("spoken_form_plausibility_review").validate(context)

    assert chunk_calls[0]["max_tokens"] == 1024


def test_validator_uses_validation_llm_concurrency_override(tmp_path, monkeypatch):
    transcript = parse_transcript_json(
        """
        [
          {"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "There were gestures at the temple."},
          {"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "ChatGPT still can't do that with a dam."}
        ]
        """
    )
    proposals = [
        CorrectionProposal(
            proposal_index=0,
            module_instance="homophones",
            module_key="homophones",
            id=1,
            original_text="gestures",
            corrected_text="Jesters",
            confidence=0.95,
        ),
        CorrectionProposal(
            proposal_index=1,
            module_instance="homophones",
            module_key="homophones",
            id=2,
            original_text="dam",
            corrected_text="damn",
            confidence=0.95,
        ),
    ]
    client = CoordinatedStructuredLLMClient(
        [
            lambda **kwargs: {
                "validations": [
                    {
                        "correction_index": 0 if '"correction_index": 0' in kwargs["messages"][1]["content"] else 1,
                        "approved": True,
                        "confidence": 0.97,
                        "reason": "ok",
                    }
                ]
            },
            lambda **kwargs: {
                "validations": [
                    {
                        "correction_index": 0 if '"correction_index": 0' in kwargs["messages"][1]["content"] else 1,
                        "approved": True,
                        "confidence": 0.94,
                        "reason": "ok",
                    }
                ]
            },
        ],
        threading.Barrier(2, timeout=1.0),
    )

    def fake_chunk_payload_items(items, max_tokens, payload_fn, empty_error_message):
        return [
            TokenBatch(batch_index=0, items=list(items[:1]), token_count=1),
            TokenBatch(batch_index=1, items=list(items[1:]), token_count=1),
        ]

    monkeypatch.setattr(llm_module, "chunk_payload_items", fake_chunk_payload_items)

    result = SpokenFormPlausibilityValidator("spoken_form_plausibility_review").validate(
        _context(
            proposals=proposals,
            transcript=transcript,
            llm_client=client,
            tmp_path=tmp_path,
            llm_concurrency=1,
            validation_llm_concurrency=2,
        )
    )

    assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [
        (0, True),
        (1, True),
    ]
    assert len(client.calls) == 2