Complete assembled validation and migrate remaining tests

This commit is contained in:
2026-11-10 18:13:12 +00:00
parent ba1d5d2367
commit 2844ce7e6a
44 changed files with 577 additions and 1624 deletions

View File

@@ -5,8 +5,8 @@ import (
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
@@ -15,13 +15,13 @@ import (
"testing"
"time"
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
pk "gitea.maximumdirect.net/eric/promptkit"
ck "go.maximumdirect.net/configkit"
pk "go.maximumdirect.net/promptkit"
)
type reply struct {
@@ -30,85 +30,56 @@ type reply struct {
}
func adapterFixture(t *testing.T, replies []reply, retries, repairs int, hook func(*http.Request)) (*Runtime, *atomic.Int32, string) {
return adapterFixtureWithDocuments(t, replies, retries, repairs, hook, "offline", map[string]string{
"definitions.yaml": "profiles:\n base:\n backend: offline\n model: full-model\n",
})
return adapterFixtureWithPrompt(t, replies, retries, repairs, hook, "")
}
func adapterFixtureWithPrompt(t *testing.T, replies []reply, retries, repairs int, hook func(*http.Request), operatorPrompt string) (*Runtime, *atomic.Int32, string) {
documents := map[string]string{
"definitions.yaml": "profiles:\n base:\n backend: offline\n model: full-model\n",
}
if operatorPrompt != "" {
documents["operator-prompt.yaml"] = operatorPrompt
}
return adapterFixtureWithDocuments(t, replies, retries, repairs, hook, "offline", documents)
}
func adapterFixtureWithDocuments(t *testing.T, replies []reply, retries, repairs int, hook func(*http.Request), backendID string, documents map[string]string) (*Runtime, *atomic.Int32, string) {
t.Helper()
var count atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
t.Setenv("ADAPTER_KEY", "adapter-secret")
dir := t.TempDir()
resources := "backends:\n - id: offline\n endpoint: https://provider.invalid/v1\n api_key_env: ADAPTER_KEY\n concurrency_limit: 1\n queue_capacity: 0\nprofiles:\n - id: base\n backend: offline\n model: full-model\n"
for name, body := range map[string]string{"config.yml": "version: 3\n", "prompts.yml": resources + operatorPrompt} {
if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0600); err != nil {
t.Fatal(err)
}
}
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
if err != nil {
t.Fatal(err)
}
transport := transportFunc(func(r *http.Request) (*http.Response, error) {
call := int(count.Add(1))
if hook != nil {
hook(r)
}
step := replies[len(replies)-1]
if call <= len(replies) {
step = replies[call-1]
if err := r.Context().Err(); err != nil {
return nil, err
}
step := replies[min(call-1, len(replies)-1)]
if step.status == 0 {
connection, _, hijackErr := w.(http.Hijacker).Hijack()
if hijackErr != nil {
t.Error(hijackErr)
return
return nil, errors.New("synthetic transport error")
}
body := `{"error":{"message":"provider error"}}`
if step.status == http.StatusOK {
body = `{"choices":[]}`
if step.content != "MALFORMED_ENVELOPE" {
encoded, _ := json.Marshal(step.content)
body = fmt.Sprintf(`{"model":"returned-model","choices":[{"message":{"content":%s}}],"usage":{"prompt_tokens":2,"completion_tokens":3,"total_tokens":5}}`, encoded)
}
_ = connection.Close()
return
}
if step.status != http.StatusOK {
w.WriteHeader(step.status)
_, _ = fmt.Fprint(w, `{"error":{"message":"provider error"}}`)
return
}
if step.content == "MALFORMED_ENVELOPE" {
_, _ = fmt.Fprint(w, `{"choices":[]}`)
return
}
encoded, _ := json.Marshal(step.content)
w.Header().Set("Content-Type", "application/json")
_, _ = fmt.Fprintf(w, `{"model":"returned-model","choices":[{"message":{"content":%s}}],"usage":{"prompt_tokens":2,"completion_tokens":3,"total_tokens":5}}`, encoded)
}))
t.Cleanup(server.Close)
t.Setenv("ADAPTER_KEY", "adapter-secret")
dir := t.TempDir()
profileDirectory := filepath.Join(dir, "profiles")
if err := os.Mkdir(profileDirectory, 0700); err != nil {
t.Fatal(err)
}
base := fmt.Sprintf("version: 2\nprofiles_dir: profiles\nbackends:\n %s:\n endpoint: %s/v1\n credential: ADAPTER_KEY\n concurrency: 1\n queue_capacity: 0\nllm:\n proposal_profile: base\n transient_retries: %d\n repair_attempts: %d\n", backendID, server.URL, retries, repairs)
main := filepath.Join(dir, "config.yml")
if err := os.WriteFile(main, []byte(base), 0600); err != nil {
t.Fatal(err)
}
for name, content := range documents {
if err := os.WriteFile(filepath.Join(profileDirectory, name), []byte(content), 0600); err != nil {
t.Fatal(err)
}
}
loaded, err := config.Load(ck.File(main), "")
if err != nil {
t.Fatal(err)
}
runtime, err := NewRuntime(context.Background(), loaded.Config, main, server.Client())
return &http.Response{StatusCode: step.status, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
})
runtime, err := NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
if err != nil {
t.Fatal(err)
}
runtime.Config.LLM = config.LLM{OperationTimeout: 10 * time.Second, TransientRetries: retries, RepairAttempts: repairs}
return runtime, &count, dir
}
func operatorPromptDocument(system, user string) string {
return fmt.Sprintf("prompts:\n %s:\n v1:\n messages:\n - role: system\n content: %q\n - role: user\n content: %q\n output:\n format: text\n validation_mode: basic\n", prompts.PromptIDModuleGlossaryProposal, system, user)
return fmt.Sprintf("collections:\n audita:\n prompts:\n - id: modules.glossary.proposal\n messages:\n - role: system\n content: %q\n - role: user\n content: %q\n output:\n format: text\n validation_mode: none\n", system, user)
}
func adapterRequest(schema responseschema.Key) structuredoutput.Request {
@@ -118,7 +89,7 @@ func adapterRequest(schema responseschema.Key) structuredoutput.Request {
id = prompts.PromptIDValidatorEditorialReview
vars = map[string]string{"TranscriptDescriptionBlock": "", "PayloadJSON": "[]"}
}
return structuredoutput.Request{PromptID: id, PromptVersion: prompts.VersionV1, Variables: vars, Schema: schema, Context: structuredoutput.DiagnosticContext{ModuleKey: "glossary"}}
return structuredoutput.Request{PromptID: id, Variables: vars, Schema: schema, Context: structuredoutput.DiagnosticContext{ModuleKey: "glossary"}}
}
func TestPreparedAdapterRepairsAndClassifies(t *testing.T) {
@@ -154,7 +125,7 @@ func TestPreparedAdapterRepairsAndClassifies(t *testing.T) {
}
var events []pk.ExecutionEvent
var records []structuredoutput.CallRecord
client, err := runtime.NewRoleClient(role, CallHooks{OnEvent: func(e pk.ExecutionEvent) { events = append(events, e) }, OnCall: func(r structuredoutput.CallRecord) { records = append(records, r) }})
client, err := runtime.NewRoleClient(role, "base", runtime.Config.LLM, CallHooks{OnEvent: func(e pk.ExecutionEvent) { events = append(events, e) }, OnCall: func(r structuredoutput.CallRecord) { records = append(records, r) }})
if err != nil {
t.Fatal(err)
}
@@ -234,7 +205,7 @@ func TestCorrectionSchemaReplacementValuesThroughAdapter(t *testing.T) {
}
payload := `{"corrections":[{` + strings.Join(fields, ",") + `}]}`
runtime, count, _ := adapterFixture(t, []reply{{200, payload}}, 0, tc.repairs, nil)
client, err := runtime.NewRoleClient("proposal", CallHooks{})
client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{})
if err != nil {
t.Fatal(err)
}
@@ -280,7 +251,7 @@ func TestPreparedAdapterMetadataAndTargetGuard(t *testing.T) {
t.Error(err)
}
})
client, err := runtime.NewRoleClient("proposal", CallHooks{})
client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{})
if err != nil {
t.Fatal(err)
}
@@ -314,12 +285,7 @@ func TestPreparedAdapterMetadataAndTargetGuard(t *testing.T) {
if !found {
t.Fatal("hardening policy missing")
}
backend := runtime.Config.Backends["offline"]
backend.Endpoint = "http://127.0.0.1:1/v1"
runtime.Config.Backends["offline"] = backend
if _, err := client.CompleteStructured(context.Background(), req, &out); err == nil || !strings.Contains(err.Error(), "target changed") || count.Load() != 1 {
t.Fatalf("changed application backend target dispatched: %v, %d", err, count.Load())
}
}
func TestOperatorPromptOverridesUsePerCallSchemaAndOptionalSharedText(t *testing.T) {
@@ -365,7 +331,7 @@ func TestOperatorPromptOverridesUsePerCallSchemaAndOptionalSharedText(t *testing
}
posted = append(posted, envelope)
}, operatorPromptDocument(tc.system, tc.user))
client, err := runtime.NewRoleClient("proposal", CallHooks{})
client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{})
if err != nil {
t.Fatal(err)
}
@@ -403,150 +369,9 @@ func TestOperatorPromptOverridesUsePerCallSchemaAndOptionalSharedText(t *testing
}
}
func TestProfilePresenceBackendAuthorityAndProviderParametersReachWire(t *testing.T) {
t.Setenv("CATALOG_TRAP_KEY", "catalog-secret-must-not-be-used")
cases := []struct {
name string
parentTemperature string
childTemperature string
wantTemperature bool
}{
{name: "explicit zero", parentTemperature: " temperature: 0.8\n", childTemperature: " temperature: 0\n", wantTemperature: true},
{name: "omitted", wantTemperature: false},
{name: "null clears inherited control", parentTemperature: " temperature: 0.8\n", childTemperature: " temperature: null\n", wantTemperature: false},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
profileDocument := "profiles:\n \"parent~/partial\":\n backend: openrouter\n model: parent-model\n" + tc.parentTemperature +
" base:\n extends: \"parent~/partial\"\n model: full-model\n timeout_seconds: 0\n" + tc.childTemperature +
" extra_params:\n routing:\n overridden: profile\n profile_only: retained\n cleared: null\n"
operatorBackend := `backends:
openrouter:
endpoint: https://operator.invalid/v1
concurrency_limit: 91
queue_capacity: 92
api_key_env: CATALOG_TRAP_KEY
extra_params:
routing:
inherited: backend
overridden: backend
cleared: remove-me
`
documents := map[string]string{
"unrelated-profile-file.yml": profileDocument,
"provider-tuning.yaml": operatorBackend,
}
var wire map[string]json.RawMessage
runtime, count, _ := adapterFixtureWithDocuments(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, func(r *http.Request) {
if got := r.Header.Get("Authorization"); got != "Bearer adapter-secret" {
t.Errorf("application credential was not authoritative: %q", got)
}
if err := json.NewDecoder(r.Body).Decode(&wire); err != nil {
t.Error(err)
}
}, "openrouter", documents)
profile := runtime.Profiles["proposal"]
if profile.EffectiveModelParams.APIKeyEnv != "" || profile.EffectiveModelParams.Endpoint == "https://operator.invalid/v1" || profile.EffectiveModelParams.TimeoutSeconds != 0 {
t.Fatalf("application backend mapping or zero timeout lost: %+v", profile.EffectiveModelParams)
}
client, err := runtime.NewRoleClient("proposal", CallHooks{})
if err != nil {
t.Fatal(err)
}
var out map[string]any
response, err := client.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &out)
if err != nil {
t.Fatal(err)
}
if count.Load() != 1 {
t.Fatalf("provider calls = %d, want 1", count.Load())
}
temperature, hasTemperature := wire["temperature"]
if hasTemperature != tc.wantTemperature {
t.Fatalf("temperature presence = %t, want %t: %s", hasTemperature, tc.wantTemperature, wire)
}
if tc.wantTemperature {
var value float64
if err := json.Unmarshal(temperature, &value); err != nil || value != 0 {
t.Fatalf("explicit zero temperature = %s (%v)", temperature, err)
}
}
var routing map[string]any
if err := json.Unmarshal(wire["routing"], &routing); err != nil {
t.Fatalf("provider routing fields missing: %s (%v)", wire, err)
}
if routing["inherited"] != "backend" || routing["overridden"] != "profile" || routing["profile_only"] != "retained" {
t.Fatalf("recursively overlaid provider fields = %#v", routing)
}
if value, exists := routing["cleared"]; !exists || value != nil {
t.Fatalf("null did not replace inherited provider field: %#v", routing)
}
if response.Model != "full-model" || response.Prepared == nil {
t.Fatalf("execution did not use selected profile: %+v", response)
}
})
}
}
func TestPreparedAdapterUsesCapturedProfileAndPromptUntilReconstructed(t *testing.T) {
var posted []struct {
Model string `json:"model"`
Messages []structuredoutput.Message `json:"messages"`
}
runtime, count, dir := adapterFixtureWithPrompt(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, func(r *http.Request) {
var envelope struct {
Model string `json:"model"`
Messages []structuredoutput.Message `json:"messages"`
}
if err := json.NewDecoder(r.Body).Decode(&envelope); err != nil {
t.Error(err)
return
}
posted = append(posted, envelope)
}, operatorPromptDocument("Initial operator instructions.", "Initial payload."))
client, err := runtime.NewRoleClient("proposal", CallHooks{})
if err != nil {
t.Fatal(err)
}
profile := "profiles:\n base:\n backend: offline\n model: edited-model\n"
if err := os.WriteFile(filepath.Join(dir, "profiles", "definitions.yaml"), []byte(profile), 0600); err != nil {
t.Fatal(err)
}
updatedPrompt := operatorPromptDocument("Edited operator instructions.", "Edited payload.")
if err := os.WriteFile(filepath.Join(dir, "profiles", "operator-prompt.yaml"), []byte(updatedPrompt), 0600); err != nil {
t.Fatal(err)
}
var out map[string]any
response, err := client.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &out)
if err != nil {
t.Fatal(err)
}
if response.Model != "full-model" || response.Prepared == nil || response.Prepared.Messages[0].Content != "Initial operator instructions." || posted[0].Model != "full-model" {
t.Fatalf("existing runtime did not retain its captured configuration: response=%+v request=%+v", response, posted[0])
}
main := filepath.Join(dir, "config.yml")
newRuntime, err := NewRuntime(context.Background(), runtime.Config, main, http.DefaultClient)
if err != nil {
t.Fatal(err)
}
newClient, err := newRuntime.NewRoleClient("proposal", CallHooks{})
if err != nil {
t.Fatal(err)
}
response, err = newClient.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &out)
if err != nil {
t.Fatal(err)
}
if int(count.Load()) != 2 || response.Model != "edited-model" || response.Prepared == nil || response.Prepared.Messages[0].Content != "Edited operator instructions." || posted[1].Model != "edited-model" {
t.Fatalf("new runtime did not capture edited configuration: response=%+v request=%+v", response, posted[1])
}
}
func TestPreparedAdapterRejectsTargetDriftAndPreparationErrors(t *testing.T) {
runtime, count, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, nil)
client, err := runtime.NewRoleClient("proposal", CallHooks{})
client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{})
if err != nil {
t.Fatal(err)
}
@@ -563,7 +388,7 @@ func TestPreparedAdapterCancellationStopsRetryWait(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
var once sync.Once
client, err := runtime.NewRoleClient("proposal", CallHooks{OnEvent: func(e pk.ExecutionEvent) {
client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{OnEvent: func(e pk.ExecutionEvent) {
if e.Kind == pk.ExecutionAttempt && e.Dispatched {
once.Do(cancel)
}
@@ -580,11 +405,9 @@ func TestPreparedAdapterCancellationStopsRetryWait(t *testing.T) {
func TestPreparedAdapterDistinguishesOperationTimeoutFromParentCancellation(t *testing.T) {
blocking := func(*http.Request) { time.Sleep(500 * time.Millisecond) }
runtime, _, _ := adapterFixtureWithDocuments(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, blocking, "offline", map[string]string{
"definitions.yaml": "profiles:\n base:\n backend: offline\n model: full-model\n timeout_seconds: 0\n",
})
runtime, _, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, blocking)
runtime.Config.LLM.OperationTimeout = 200 * time.Millisecond
client, err := runtime.NewRoleClient("proposal", CallHooks{})
client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{})
if err != nil {
t.Fatal(err)
}
@@ -605,7 +428,7 @@ func TestPreparedAdapterDistinguishesOperationTimeoutFromParentCancellation(t *t
func TestPreparedAdapterClassifiesSetupAndDomainDecodeFailures(t *testing.T) {
runtime, _, _ := adapterFixture(t, []reply{{200, `{"validations":[{"correction_index":9999999999999999999999999,"approved":true,"confidence":0.8,"reason":"ok"}]}`}}, 0, 0, nil)
client, err := runtime.NewRoleClient("validation", CallHooks{})
client, err := runtime.NewRoleClient("validation", "base", runtime.Config.LLM, CallHooks{})
if err != nil {
t.Fatal(err)
}
@@ -638,7 +461,7 @@ func TestPreparedAdapterClassifiesSetupAndDomainDecodeFailures(t *testing.T) {
func TestPreparedAdapterNonRetryableProviderStatus(t *testing.T) {
runtime, count, _ := adapterFixture(t, []reply{{http.StatusBadRequest, ""}}, 2, 0, nil)
client, err := runtime.NewRoleClient("proposal", CallHooks{})
client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{})
if err != nil {
t.Fatal(err)
}