eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Complete assembled validation and migrate remaining tests

Browse Source
This commit is contained in:
Eric Rakestraw
2026-11-10 18:13:12 +00:00
parent ba1d5d2367
commit 2844ce7e6a
44 changed files with 577 additions and 1624 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
internal/framework/validators/llm_integration_test.go
View File

@@ -4,19 +4,24 @@ import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
ck "go.maximumdirect.net/configkit"
)
type validationTransport func(*http.Request) (*http.Response, error)
func (f validationTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) {
valid := `{"validations":[{"correction_index":1,"approved":true,"confidence":0.9,"reason":"ok"}]}`
for _, tc := range []struct {
@@ -32,41 +37,38 @@ func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) {
} {
t.Run(tc.name, func(t *testing.T) {
calls := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
transport := validationTransport(func(r *http.Request) (*http.Response, error) {
calls++
output := tc.responses[len(tc.responses)-1]
if calls <= len(tc.responses) {
output = tc.responses[calls-1]
}
if output == "MALFORMED_ENVELOPE" {
_, _ = fmt.Fprint(w, `{"choices":[]}`)
return
body := `{"choices":[]}`
if output != "MALFORMED_ENVELOPE" {
content, _ := json.Marshal(output)
body = fmt.Sprintf(`{"choices":[{"message":{"content":%s}}]}`, content)
}
content, _ := json.Marshal(output)
_, _ = fmt.Fprintf(w, `{"choices":[{"message":{"content":%s}}]}`, content)
}))
defer server.Close()
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
})
dir := t.TempDir()
if err := os.Mkdir(filepath.Join(dir, "profiles"), 0700); err != nil {
t.Fatal(err)
}
main := filepath.Join(dir, "config.yml")
content := fmt.Sprintf("version: 2\nbackends:\n offline:\n endpoint: %s/v1\nllm:\n proposal_profile: base\n transient_retries: 0\n repair_attempts: 1\n", server.URL)
if err := os.WriteFile(main, []byte(content), 0600); err != nil {
if err := os.WriteFile(main, []byte("version: 3\n"), 0600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "profiles", "base.yaml"), []byte("profiles:\n base:\n backend: offline\n model: local-model\n"), 0600); err != nil {
if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte("profiles:\n - {id: base, endpoint: 'https://example.invalid/v1', model: local-model, api_key_required: false}\n"), 0600); err != nil {
t.Fatal(err)
}
loaded, err := config.Load(ck.File(main), "")
loaded, err := config.Load(ck.Input{Source: ck.File(main)}, "")
if err != nil {
t.Fatal(err)
}
runtime, err := llm.NewRuntime(context.Background(), loaded.Config, main, server.Client())
runtime, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
if err != nil {
t.Fatal(err)
}
client, err := runtime.NewRoleClient("validation", llm.CallHooks{})
settings := loaded.Config.LLM
settings.TransientRetries, settings.RepairAttempts = 0, 1
client, err := runtime.NewRoleClient("validation", "base", settings, llm.CallHooks{})
if err != nil {
t.Fatal(err)
}
@@ -75,7 +77,6 @@ func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) {
t.Fatal(err)
}
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.Config = &loaded.Config
req.LLMClient = client
result, err := validator.Validate(context.Background(), req)
if calls != tc.wantCalls {