audita/internal/framework/validators/llm_validators_test.go

package validators

import (
	"context"
	"errors"
	"strings"
	"testing"

	"gitea.maximumdirect.net/eric/audita/internal/core/chunking"
	"gitea.maximumdirect.net/eric/audita/internal/core/config"
	"gitea.maximumdirect.net/eric/audita/internal/core/schema"
	"gitea.maximumdirect.net/eric/audita/internal/framework/outcomes"
	"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
	"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
	"gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput"
	"gitea.maximumdirect.net/eric/audita/internal/prompts"
)

type fakeStructuredLLMClient struct {
	responses []LLMValidationResponse
	err       error
	calls     []StructuredCompletionRequest
}

type captureValidationDiagnosticsWriter struct {
	err                 error
	lastRequestMetadata any
}

func (w *captureValidationDiagnosticsWriter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error) {
	_ = stage
	_ = requestPayload
	_ = responsePayload
	_ = errorPayload
	w.lastRequestMetadata = requestMetadata
	return InteractionArtifacts{}, w.err
}

func (f *fakeStructuredLLMClient) CompleteStructured(ctx context.Context, req StructuredCompletionRequest, out any) (StructuredCompletionResponse, error) {
	_ = ctx
	f.calls = append(f.calls, req)
	if f.err != nil {
		return StructuredCompletionResponse{}, f.err
	}
	if len(f.responses) == 0 {
		return StructuredCompletionResponse{}, errors.New("unexpected call")
	}
	resp := f.responses[0]
	f.responses = f.responses[1:]
	target, ok := out.(*LLMValidationResponse)
	if !ok {
		return StructuredCompletionResponse{}, errors.New("unexpected output type")
	}
	*target = resp
	return StructuredCompletionResponse{}, nil
}

func makeReq(candidates []proposals.EnrichedCorrectionProposal) Request {
	cfg := config.Config{}
	cfg.Validation.MaxPromptTokens = 10000
	return Request{
		WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple.", Categories: []string{"narration"}}}},
		CandidateProposal: candidates,
		ModuleKey:         "homophones",
		ModuleInstance:    "homophones",
		ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
		Config:            &cfg,
	}
}

func mk(index int, orig, corr string) proposals.EnrichedCorrectionProposal {
	return proposals.EnrichedCorrectionProposal{
		CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: orig, CorrectedText: corr, Confidence: 0.9},
		ProposalMetadata:   proposals.ProposalMetadata{ProposalIndex: index, ModuleKey: "homophones", ModuleInstance: "homophones"},
	}
}

func TestChunkLLMValidationItemsOneSmallBatch(t *testing.T) {
	items := []LLMValidationItem{{CorrectionIndex: 0, OriginalText: "a", CorrectedText: "b", OriginalSegmentText: "x", CorrectedSegmentText: "y"}}
	batches, err := ChunkLLMValidationItems(items, 1000, chunking.NewSimpleTokenEstimator())
	if err != nil {
		t.Fatalf("Chunk error: %v", err)
	}
	if len(batches) != 1 {
		t.Fatalf("expected 1 batch, got %d", len(batches))
	}
}

func TestChunkLLMValidationItemsMultipleBatchesStableNoDropNoDup(t *testing.T) {
	items := []LLMValidationItem{
		{CorrectionIndex: 0, OriginalText: strings.Repeat("a", 20), CorrectedText: "b", OriginalSegmentText: "x", CorrectedSegmentText: "y"},
		{CorrectionIndex: 1, OriginalText: strings.Repeat("c", 20), CorrectedText: "d", OriginalSegmentText: "x", CorrectedSegmentText: "y"},
		{CorrectionIndex: 2, OriginalText: strings.Repeat("e", 20), CorrectedText: "f", OriginalSegmentText: "x", CorrectedSegmentText: "y"},
	}
	batches, err := ChunkLLMValidationItems(items, 40, chunking.NewSimpleTokenEstimator())
	if err != nil {
		t.Fatalf("Chunk error: %v", err)
	}
	if len(batches) < 2 {
		t.Fatalf("expected multiple batches, got %d", len(batches))
	}
	seen := make([]int, 0)
	for _, b := range batches {
		for _, it := range b.Items {
			seen = append(seen, it.CorrectionIndex)
		}
	}
	if len(seen) != 3 || seen[0] != 0 || seen[1] != 1 || seen[2] != 2 {
		t.Fatalf("unexpected ordering/drops/dups: %v", seen)
	}
}

func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) {
	client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
		{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
		{CorrectionIndex: 1, Approved: false, Confidence: 0.95, Reason: "bad"},
	}}}}
	v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
	if err != nil {
		t.Fatalf("new validator error: %v", err)
	}
	req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters"), mk(1, "gestures", "Lyra")})
	req.LLMClient = client
	res, err := v.Validate(context.Background(), req)
	if err != nil {
		t.Fatalf("validate err: %v", err)
	}
	if len(res.Decisions) != 2 || !res.Decisions[0].Approved || res.Decisions[1].Approved {
		t.Fatalf("unexpected decisions: %+v", res.Decisions)
	}
	if len(client.calls) != 1 {
		t.Fatalf("expected one LLM call, got %d", len(client.calls))
	}
	call := client.calls[0]
	if call.Schema != responseschema.ValidatorDecisionSetKey || call.PromptID != prompts.PromptIDValidatorSpokenFormPlausibility || call.Context.Role != "validation" || call.Context.ValidatorName != v.Name() || !strings.Contains(call.Variables["PayloadJSON"], "correction_index") {
		t.Fatalf("unexpected validator call: %+v", call)
	}
}

func TestLLMBackedValidatorMalformedOutputRejectsBatch(t *testing.T) {
	client := &fakeStructuredLLMClient{err: &structuredoutput.Failure{Class: structuredoutput.FailureRecoverable, Reason: outcomes.ReasonInvalidOutputExhausted, Message: "structured output remained invalid"}}
	v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
	req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
	req.LLMClient = client
	res, err := v.Validate(context.Background(), req)
	if err != nil {
		t.Fatalf("expected malformed output downgrade, got %v", err)
	}
	if len(res.Decisions) != 1 || res.Decisions[0].Approved || res.Decisions[0].Disposition != DispositionUnavailable || res.Decisions[0].ReasonCode != string(outcomes.ReasonInvalidOutputExhausted) {
		t.Fatalf("unexpected decisions: %+v", res.Decisions)
	}
	if len(res.Operations) != 1 || res.Operations[0].FailureReason != outcomes.ReasonInvalidOutputExhausted {
		t.Fatalf("expected failed operation, got %+v", res.Operations)
	}
}

func TestLLMBackedValidatorProviderMalformedEnvelopePropagates(t *testing.T) {
	client := &fakeStructuredLLMClient{err: &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "structured model call failed", Cause: errors.New("malformed provider envelope")}}
	v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
	req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
	req.LLMClient = client
	result, err := v.Validate(context.Background(), req)
	if err == nil || len(result.Operations) != 1 || result.Operations[0].FailureReason != outcomes.ReasonInternalFailure {
		t.Fatalf("protocol error was downgraded: %+v, %v", result, err)
	}
}

func TestLLMBackedValidatorMissingDecisionRejectsBatch(t *testing.T) {
	client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{}}}}
	v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
	req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
	req.LLMClient = client
	res, err := v.Validate(context.Background(), req)
	if err != nil {
		t.Fatalf("expected missing decision downgrade, got %v", err)
	}
	if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != string(outcomes.ReasonValidationDecisionsInvalid) || res.Decisions[0].Disposition != DispositionUnavailable {
		t.Fatalf("unexpected decisions: %+v", res.Decisions)
	}
}

func TestLLMBackedValidatorDuplicateDecisionRejectsBatch(t *testing.T) {
	client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
		{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
		{CorrectionIndex: 0, Approved: false, Confidence: 0.9, Reason: "dup"},
	}}}}
	v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
	req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
	req.LLMClient = client
	res, err := v.Validate(context.Background(), req)
	if err != nil {
		t.Fatalf("expected duplicate decision downgrade, got %v", err)
	}
	if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != string(outcomes.ReasonValidationDecisionsInvalid) || res.Decisions[0].Disposition != DispositionUnavailable {
		t.Fatalf("unexpected decisions: %+v", res.Decisions)
	}
}

func TestLLMBackedValidatorUnknownProposalIndexRejectsBatch(t *testing.T) {
	client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 99, Approved: true, Confidence: 0.9, Reason: "unknown"}}}}}
	v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
	req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
	req.LLMClient = client
	res, err := v.Validate(context.Background(), req)
	if err != nil {
		t.Fatalf("expected unknown index downgrade, got %v", err)
	}
	if len(res.Decisions) != 1 || res.Decisions[0].ReasonCode != string(outcomes.ReasonValidationDecisionsInvalid) || res.Decisions[0].Disposition != DispositionUnavailable {
		t.Fatalf("unexpected decisions: %+v", res.Decisions)
	}
}

func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *testing.T) {
	client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
		{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
	}}}}
	v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
	huge := strings.Repeat("gestures ", 200)
	req := Request{
		WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{
			{ID: 1, Text: huge},
			{ID: 2, Text: "There were gestures at the temple.", Categories: []string{"narration"}},
		}},
		CandidateProposal: []proposals.EnrichedCorrectionProposal{
			{
				CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: huge, CorrectedText: "Jesters", Confidence: 0.9},
				ProposalMetadata:   proposals.ProposalMetadata{ProposalIndex: 0, ModuleKey: "homophones", ModuleInstance: "homophones"},
			},
			{
				CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 2, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.9},
				ProposalMetadata:   proposals.ProposalMetadata{ProposalIndex: 1, ModuleKey: "homophones", ModuleInstance: "homophones"},
			},
		},
		ModuleKey:         "homophones",
		ModuleInstance:    "homophones",
		ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
	}
	req.LLMClient = client
	cfg := config.Config{}
	cfg.Validation.MaxPromptTokens = 200
	req.Config = &cfg

	res, err := v.Validate(context.Background(), req)
	if err != nil {
		t.Fatalf("expected oversize downgrade, got %v", err)
	}
	if len(res.Decisions) != 2 {
		t.Fatalf("expected two decisions, got %+v", res.Decisions)
	}
	if res.Decisions[0].ReasonCode != ReasonValidatorInputTooLarge || res.Decisions[0].Approved || res.Decisions[0].Disposition != DispositionUnavailable {
		t.Fatalf("expected first decision oversize rejection, got %+v", res.Decisions[0])
	}
	if !res.Decisions[1].Approved {
		t.Fatalf("expected second decision approved, got %+v", res.Decisions[1])
	}
	if len(res.Operations) != 2 || res.Operations[0].Reference.Kind != outcomes.OperationValidationInput || res.Operations[0].FailureReason != outcomes.ReasonValidationInputTooLarge || res.Operations[0].Reference.BatchIndex != nil || len(res.Operations[0].ProposalIndices) != 1 || res.Operations[0].ProposalIndices[0] != 0 {
		t.Fatalf("expected consolidated oversized input outcome, got %+v", res.Operations)
	}
}

func TestLLMBackedValidatorDiagnosticsIncludeSchemaMetadata(t *testing.T) {
	client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
		{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
	}}}}
	writer := &captureValidationDiagnosticsWriter{}
	v, err := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility)
	if err != nil {
		t.Fatalf("new validator error: %v", err)
	}
	req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
	req.LLMClient = client
	req.DiagnosticsWriter = writer
	_, err = v.Validate(context.Background(), req)
	if err != nil {
		t.Fatalf("validate error: %v", err)
	}
	metadata, ok := writer.lastRequestMetadata.(map[string]any)
	if !ok {
		t.Fatalf("expected metadata map, got %T", writer.lastRequestMetadata)
	}
	schemaMap, ok := metadata["response_schema"].(map[string]any)
	if !ok {
		t.Fatalf("expected response_schema map, got %T", metadata["response_schema"])
	}
	want := responseschema.MustLookup(responseschema.ValidatorDecisionSetKey)
	if schemaMap["id"] != want.ID || schemaMap["version"] != want.Version || schemaMap["name"] != want.Name || schemaMap["sha256"] != want.SHA256 {
		t.Fatalf("unexpected diagnostics schema metadata: got=%v want=%+v", schemaMap, want)
	}

}
func TestEveryValidationTypeSelectsEmbeddedPromptAndVariables(t *testing.T) {
	cases := []struct {
		kind   LLMValidatorType
		prompt string
	}{
		{LLMValidatorTypeSpokenFormPlausibility, prompts.PromptIDValidatorSpokenFormPlausibility},
		{LLMValidatorTypeMeaningReversal, prompts.PromptIDValidatorMeaningReversalReview},
		{LLMValidatorTypeEditorialReview, prompts.PromptIDValidatorEditorialReview},
		{LLMValidatorTypeGrammarReview, prompts.PromptIDValidatorGrammarReview},
		{LLMValidatorTypeSpokenWordReview, prompts.PromptIDValidatorSpokenWordReview},
	}
	for _, tc := range cases {
		t.Run(string(tc.kind), func(t *testing.T) {
			client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: .9, Reason: "ok"}}}}}
			validator, err := NewLLMBackedValidator(string(tc.kind), tc.kind)
			if err != nil {
				t.Fatal(err)
			}
			req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
			req.LLMClient = client
			req.Config.Context.Description = " Fantasy scene "
			if _, err := validator.Validate(context.Background(), req); err != nil {
				t.Fatal(err)
			}
			if len(client.calls) != 1 {
				t.Fatalf("calls=%d", len(client.calls))
			}
			call := client.calls[0]
			if call.PromptID != tc.prompt || call.PromptVersion != prompts.VersionV1 || call.Schema != responseschema.ValidatorDecisionSetKey || call.Context.Role != "validation" || call.Context.BatchIndex == nil || !strings.Contains(call.Variables["PayloadJSON"], "correction_index") || !strings.Contains(call.Variables["TranscriptDescriptionBlock"], "Fantasy scene") {
				t.Fatalf("mapping: %+v", call)
			}
		})
	}
}

func TestValidationDiagnosticFailureDoesNotChangeDecision(t *testing.T) {
	cause := errors.New("diagnostic storage unavailable")
	req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
	req.LLMClient = &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 1, Reason: "ok"}}}}}
	req.DiagnosticsWriter = &captureValidationDiagnosticsWriter{err: cause}
	var reported error
	req.ReportDiagnosticError = func(err error) { reported = err }
	validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
	result, err := validator.Validate(context.Background(), req)
	if err != nil || !errors.Is(reported, cause) || len(result.Decisions) != 1 || !result.Decisions[0].Approved || result.Decisions[0].DiagnosticArtifactPath != "" || len(result.Operations) != 1 || !result.Operations[0].Succeeded || len(result.Operations[0].RecoveryReasons) != 0 {
		t.Fatalf("debug failure changed outcome: %+v %v reported=%v", result, err, reported)
	}
}