audita/internal/framework/validators/llm_integration_test.go

package validators

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
	"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
	"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
	"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
	ck "go.maximumdirect.net/configkit"
)

type validationTransport func(*http.Request) (*http.Response, error)

func (f validationTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }

func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) {
	valid := `{"validations":[{"correction_index":1,"approved":true,"confidence":0.9,"reason":"ok"}]}`
	for _, tc := range []struct {
		name         string
		responses    []string
		wantCalls    int
		wantApproved bool
		wantFailure  outcomes.ReasonCode
	}{
		{"repair reaches decisions", []string{`{"wrong":true}`, valid}, 2, true, ""},
		{"exhausted repair rejects batch", []string{`{"wrong":true}`, `{"wrong":true}`}, 2, false, outcomes.ReasonInvalidOutputExhausted},
		{"malformed envelope excludes batch", []string{"MALFORMED_ENVELOPE"}, 1, false, outcomes.ReasonProviderResponseInvalid},
	} {
		t.Run(tc.name, func(t *testing.T) {
			calls := 0
			transport := validationTransport(func(r *http.Request) (*http.Response, error) {
				calls++
				output := tc.responses[len(tc.responses)-1]
				if calls <= len(tc.responses) {
					output = tc.responses[calls-1]
				}
				body := `{"choices":[]}`
				if output != "MALFORMED_ENVELOPE" {
					content, _ := json.Marshal(output)
					body = fmt.Sprintf(`{"choices":[{"message":{"content":%s}}]}`, content)
				}
				return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
			})
			dir := t.TempDir()
			main := filepath.Join(dir, "config.yml")
			if err := os.WriteFile(main, []byte("version: 3\n"), 0600); err != nil {
				t.Fatal(err)
			}
			if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte("profiles:\n  - {id: base, endpoint: 'https://example.invalid/v1', model: local-model, api_key_required: false}\n"), 0600); err != nil {
				t.Fatal(err)
			}
			loaded, err := config.Load(ck.Input{Source: ck.File(main)}, "")
			if err != nil {
				t.Fatal(err)
			}
			runtime, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
			if err != nil {
				t.Fatal(err)
			}
			settings := loaded.Config.LLM
			settings.TransientRetries, settings.RepairAttempts = 0, 1
			client, err := runtime.NewRoleClient("validation", "base", settings, llm.CallHooks{})
			if err != nil {
				t.Fatal(err)
			}
			validator, err := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
			if err != nil {
				t.Fatal(err)
			}
			req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
			req.LLMClient = client
			result, err := validator.Validate(context.Background(), req)
			if calls != tc.wantCalls {
				t.Fatalf("calls=%d, want %d", calls, tc.wantCalls)
			}
			if err != nil {
				t.Fatal(err)
			}
			if len(result.Decisions) != 1 || result.Decisions[0].Approved != tc.wantApproved || len(result.Operations) != 1 || result.Operations[0].FailureReason != tc.wantFailure {
				t.Fatalf("domain result: %+v", result)
			}
		})
	}
}