package validators import ( "context" "encoding/json" "fmt" "io" "net/http" "os" "path/filepath" "strings" "testing" "gitea.maximumdirect.net/eric/audita/v3/internal/core/config" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput" ck "go.maximumdirect.net/configkit" ) type validationTransport func(*http.Request) (*http.Response, error) func (f validationTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) { valid := `{"validations":[{"correction_index":1,"approved":true,"reason":"ok"}]}` for _, tc := range []struct { name string responses []string wantCalls int wantApproved bool wantFailure outcomes.ReasonCode }{ {"repair reaches decisions", []string{`{"wrong":true}`, valid}, 2, true, ""}, {"exhausted repair rejects batch", []string{`{"wrong":true}`, `{"wrong":true}`}, 2, false, outcomes.ReasonInvalidOutputExhausted}, {"malformed envelope excludes batch", []string{"MALFORMED_ENVELOPE"}, 1, false, outcomes.ReasonProviderResponseInvalid}, } { t.Run(tc.name, func(t *testing.T) { calls := 0 transport := validationTransport(func(r *http.Request) (*http.Response, error) { calls++ output := tc.responses[len(tc.responses)-1] if calls <= len(tc.responses) { output = tc.responses[calls-1] } body := `{"choices":[]}` if output != "MALFORMED_ENVELOPE" { content, _ := json.Marshal(output) body = fmt.Sprintf(`{"choices":[{"message":{"content":%s}}]}`, content) } return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil }) client := testValidationClient(t, transport) validator, err := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview) if err != nil { t.Fatal(err) } req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")}) req.LLMClient = client result, err := validator.Validate(context.Background(), req) if calls != tc.wantCalls { t.Fatalf("calls=%d, want %d", calls, tc.wantCalls) } if err != nil { t.Fatal(err) } if len(result.Decisions) != 1 || result.Decisions[0].Approved != tc.wantApproved || len(result.Operations) != 1 || result.Operations[0].FailureReason != tc.wantFailure { t.Fatalf("domain result: %+v", result) } }) } } // The transport supplies provider responses; PromptKit performs structured // validation and Audita decodes the same response through the role client. func testValidationClient(t *testing.T, transport http.RoundTripper) structuredoutput.Client { t.Helper() dir := t.TempDir() main := filepath.Join(dir, "config.yml") if err := os.WriteFile(main, []byte("version: 3\n"), 0600); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte("profiles:\n - {id: base, endpoint: 'https://example.invalid/v1', model: local-model, api_key_required: false}\n"), 0600); err != nil { t.Fatal(err) } loaded, err := config.Load(ck.Input{Source: ck.File(main)}, "") if err != nil { t.Fatal(err) } runtime, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport}) if err != nil { t.Fatal(err) } settings := loaded.Config.LLM settings.TransientRetries, settings.RepairAttempts = 0, 1 client, err := runtime.NewRoleClient("validation", "base", settings, llm.CallHooks{}) if err != nil { t.Fatal(err) } return client } func TestValidationDecodesExactReferencesBeforeReconciliation(t *testing.T) { for _, tc := range []struct { name, response string indices []int want []DecisionDisposition reasons []string failure outcomes.ReasonCode }{ { name: "usable decision beside extra", response: `{"validations":[{"correction_index":1.00e0,"approved":true,"reason":"accept"},{"correction_index":9223372036854775808,"approved":true,"reason":"extra"}]}`, indices: []int{7}, want: []DecisionDisposition{DispositionApproved}, reasons: []string{ReasonApproved}, }, { name: "usable siblings beside duplicate and omission", response: `{"validations":[{"correction_index":1e0,"approved":true,"reason":"accept"},{"correction_index":2,"approved":true,"reason":"duplicate"},{"correction_index":20e-1,"approved":false,"reason":"duplicate"},{"correction_index":4.000,"approved":false,"reason":"reject"},{"correction_index":9223372036854775808,"approved":true,"reason":"extra"}]}`, indices: []int{7, 11, 14, 23}, want: []DecisionDisposition{DispositionApproved, DispositionUnavailable, DispositionUnavailable, DispositionRejected}, reasons: []string{ReasonApproved, ReasonDuplicateValidationDecision, ReasonMissingValidationDecision, "llm_rejected"}, failure: outcomes.ReasonValidationDecisionsInvalid, }, } { t.Run(tc.name, func(t *testing.T) { calls := 0 client := testValidationClient(t, validationTransport(func(req *http.Request) (*http.Response, error) { calls++ content, _ := json.Marshal(tc.response) body := fmt.Sprintf(`{"choices":[{"message":{"content":%s}}]}`, content) return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: req}, nil })) var candidates []proposals.EnrichedCorrectionProposal for _, index := range tc.indices { candidates = append(candidates, mk(index, "gestures", "Jesters")) } validator, err := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview) if err != nil { t.Fatal(err) } sink := &captureValidationDiagnosticsWriter{} req := makeReq(candidates) req.LLMClient = client req.Interactions = sink result, err := validator.Validate(context.Background(), req) if err != nil || calls != 1 || len(result.Decisions) != len(tc.want) || len(result.Operations) != 1 { t.Fatalf("result=%+v calls=%d error=%v", result, calls, err) } for i, decision := range result.Decisions { if decision.ProposalIndex != tc.indices[i] || decision.Disposition != tc.want[i] || decision.ReasonCode != tc.reasons[i] { t.Fatalf("decision %d: %+v", i, decision) } } operation := result.Operations[0] if operation.FailureReason != tc.failure || (tc.failure == "" && (len(operation.RecoveryReasons) != 1 || operation.RecoveryReasons[0] != outcomes.RecoveryValidationDecisionsFiltered)) { t.Fatalf("operation: %+v", operation) } payload := sink.lastResponsePayload.(map[string]any) diagnostics := payload["reconciliation"].(validationReconciliation) if len(diagnostics.IgnoredIndices) != 1 || diagnostics.IgnoredIndices[0] != "9223372036854775808" || payload["final_content"] != tc.response { t.Fatalf("reference evidence changed: %+v", payload) } }) } }