Complete assembled validation and migrate remaining tests
This commit is contained in:
@@ -5,8 +5,8 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -15,13 +15,13 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
ck "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
|
||||
pk "gitea.maximumdirect.net/eric/promptkit"
|
||||
ck "go.maximumdirect.net/configkit"
|
||||
pk "go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
type reply struct {
|
||||
@@ -30,85 +30,56 @@ type reply struct {
|
||||
}
|
||||
|
||||
func adapterFixture(t *testing.T, replies []reply, retries, repairs int, hook func(*http.Request)) (*Runtime, *atomic.Int32, string) {
|
||||
return adapterFixtureWithDocuments(t, replies, retries, repairs, hook, "offline", map[string]string{
|
||||
"definitions.yaml": "profiles:\n base:\n backend: offline\n model: full-model\n",
|
||||
})
|
||||
return adapterFixtureWithPrompt(t, replies, retries, repairs, hook, "")
|
||||
}
|
||||
|
||||
func adapterFixtureWithPrompt(t *testing.T, replies []reply, retries, repairs int, hook func(*http.Request), operatorPrompt string) (*Runtime, *atomic.Int32, string) {
|
||||
documents := map[string]string{
|
||||
"definitions.yaml": "profiles:\n base:\n backend: offline\n model: full-model\n",
|
||||
}
|
||||
if operatorPrompt != "" {
|
||||
documents["operator-prompt.yaml"] = operatorPrompt
|
||||
}
|
||||
return adapterFixtureWithDocuments(t, replies, retries, repairs, hook, "offline", documents)
|
||||
}
|
||||
|
||||
func adapterFixtureWithDocuments(t *testing.T, replies []reply, retries, repairs int, hook func(*http.Request), backendID string, documents map[string]string) (*Runtime, *atomic.Int32, string) {
|
||||
t.Helper()
|
||||
var count atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
t.Setenv("ADAPTER_KEY", "adapter-secret")
|
||||
dir := t.TempDir()
|
||||
resources := "backends:\n - id: offline\n endpoint: https://provider.invalid/v1\n api_key_env: ADAPTER_KEY\n concurrency_limit: 1\n queue_capacity: 0\nprofiles:\n - id: base\n backend: offline\n model: full-model\n"
|
||||
for name, body := range map[string]string{"config.yml": "version: 3\n", "prompts.yml": resources + operatorPrompt} {
|
||||
if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
transport := transportFunc(func(r *http.Request) (*http.Response, error) {
|
||||
call := int(count.Add(1))
|
||||
if hook != nil {
|
||||
hook(r)
|
||||
}
|
||||
step := replies[len(replies)-1]
|
||||
if call <= len(replies) {
|
||||
step = replies[call-1]
|
||||
if err := r.Context().Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
step := replies[min(call-1, len(replies)-1)]
|
||||
if step.status == 0 {
|
||||
connection, _, hijackErr := w.(http.Hijacker).Hijack()
|
||||
if hijackErr != nil {
|
||||
t.Error(hijackErr)
|
||||
return
|
||||
return nil, errors.New("synthetic transport error")
|
||||
}
|
||||
body := `{"error":{"message":"provider error"}}`
|
||||
if step.status == http.StatusOK {
|
||||
body = `{"choices":[]}`
|
||||
if step.content != "MALFORMED_ENVELOPE" {
|
||||
encoded, _ := json.Marshal(step.content)
|
||||
body = fmt.Sprintf(`{"model":"returned-model","choices":[{"message":{"content":%s}}],"usage":{"prompt_tokens":2,"completion_tokens":3,"total_tokens":5}}`, encoded)
|
||||
}
|
||||
_ = connection.Close()
|
||||
return
|
||||
}
|
||||
if step.status != http.StatusOK {
|
||||
w.WriteHeader(step.status)
|
||||
_, _ = fmt.Fprint(w, `{"error":{"message":"provider error"}}`)
|
||||
return
|
||||
}
|
||||
if step.content == "MALFORMED_ENVELOPE" {
|
||||
_, _ = fmt.Fprint(w, `{"choices":[]}`)
|
||||
return
|
||||
}
|
||||
encoded, _ := json.Marshal(step.content)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = fmt.Fprintf(w, `{"model":"returned-model","choices":[{"message":{"content":%s}}],"usage":{"prompt_tokens":2,"completion_tokens":3,"total_tokens":5}}`, encoded)
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
t.Setenv("ADAPTER_KEY", "adapter-secret")
|
||||
dir := t.TempDir()
|
||||
profileDirectory := filepath.Join(dir, "profiles")
|
||||
if err := os.Mkdir(profileDirectory, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
base := fmt.Sprintf("version: 2\nprofiles_dir: profiles\nbackends:\n %s:\n endpoint: %s/v1\n credential: ADAPTER_KEY\n concurrency: 1\n queue_capacity: 0\nllm:\n proposal_profile: base\n transient_retries: %d\n repair_attempts: %d\n", backendID, server.URL, retries, repairs)
|
||||
main := filepath.Join(dir, "config.yml")
|
||||
if err := os.WriteFile(main, []byte(base), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for name, content := range documents {
|
||||
if err := os.WriteFile(filepath.Join(profileDirectory, name), []byte(content), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
loaded, err := config.Load(ck.File(main), "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runtime, err := NewRuntime(context.Background(), loaded.Config, main, server.Client())
|
||||
return &http.Response{StatusCode: step.status, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
|
||||
})
|
||||
runtime, err := NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runtime.Config.LLM = config.LLM{OperationTimeout: 10 * time.Second, TransientRetries: retries, RepairAttempts: repairs}
|
||||
return runtime, &count, dir
|
||||
}
|
||||
|
||||
func operatorPromptDocument(system, user string) string {
|
||||
return fmt.Sprintf("prompts:\n %s:\n v1:\n messages:\n - role: system\n content: %q\n - role: user\n content: %q\n output:\n format: text\n validation_mode: basic\n", prompts.PromptIDModuleGlossaryProposal, system, user)
|
||||
return fmt.Sprintf("collections:\n audita:\n prompts:\n - id: modules.glossary.proposal\n messages:\n - role: system\n content: %q\n - role: user\n content: %q\n output:\n format: text\n validation_mode: none\n", system, user)
|
||||
}
|
||||
|
||||
func adapterRequest(schema responseschema.Key) structuredoutput.Request {
|
||||
@@ -118,7 +89,7 @@ func adapterRequest(schema responseschema.Key) structuredoutput.Request {
|
||||
id = prompts.PromptIDValidatorEditorialReview
|
||||
vars = map[string]string{"TranscriptDescriptionBlock": "", "PayloadJSON": "[]"}
|
||||
}
|
||||
return structuredoutput.Request{PromptID: id, PromptVersion: prompts.VersionV1, Variables: vars, Schema: schema, Context: structuredoutput.DiagnosticContext{ModuleKey: "glossary"}}
|
||||
return structuredoutput.Request{PromptID: id, Variables: vars, Schema: schema, Context: structuredoutput.DiagnosticContext{ModuleKey: "glossary"}}
|
||||
}
|
||||
|
||||
func TestPreparedAdapterRepairsAndClassifies(t *testing.T) {
|
||||
@@ -154,7 +125,7 @@ func TestPreparedAdapterRepairsAndClassifies(t *testing.T) {
|
||||
}
|
||||
var events []pk.ExecutionEvent
|
||||
var records []structuredoutput.CallRecord
|
||||
client, err := runtime.NewRoleClient(role, CallHooks{OnEvent: func(e pk.ExecutionEvent) { events = append(events, e) }, OnCall: func(r structuredoutput.CallRecord) { records = append(records, r) }})
|
||||
client, err := runtime.NewRoleClient(role, "base", runtime.Config.LLM, CallHooks{OnEvent: func(e pk.ExecutionEvent) { events = append(events, e) }, OnCall: func(r structuredoutput.CallRecord) { records = append(records, r) }})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -234,7 +205,7 @@ func TestCorrectionSchemaReplacementValuesThroughAdapter(t *testing.T) {
|
||||
}
|
||||
payload := `{"corrections":[{` + strings.Join(fields, ",") + `}]}`
|
||||
runtime, count, _ := adapterFixture(t, []reply{{200, payload}}, 0, tc.repairs, nil)
|
||||
client, err := runtime.NewRoleClient("proposal", CallHooks{})
|
||||
client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -280,7 +251,7 @@ func TestPreparedAdapterMetadataAndTargetGuard(t *testing.T) {
|
||||
t.Error(err)
|
||||
}
|
||||
})
|
||||
client, err := runtime.NewRoleClient("proposal", CallHooks{})
|
||||
client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -314,12 +285,7 @@ func TestPreparedAdapterMetadataAndTargetGuard(t *testing.T) {
|
||||
if !found {
|
||||
t.Fatal("hardening policy missing")
|
||||
}
|
||||
backend := runtime.Config.Backends["offline"]
|
||||
backend.Endpoint = "http://127.0.0.1:1/v1"
|
||||
runtime.Config.Backends["offline"] = backend
|
||||
if _, err := client.CompleteStructured(context.Background(), req, &out); err == nil || !strings.Contains(err.Error(), "target changed") || count.Load() != 1 {
|
||||
t.Fatalf("changed application backend target dispatched: %v, %d", err, count.Load())
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestOperatorPromptOverridesUsePerCallSchemaAndOptionalSharedText(t *testing.T) {
|
||||
@@ -365,7 +331,7 @@ func TestOperatorPromptOverridesUsePerCallSchemaAndOptionalSharedText(t *testing
|
||||
}
|
||||
posted = append(posted, envelope)
|
||||
}, operatorPromptDocument(tc.system, tc.user))
|
||||
client, err := runtime.NewRoleClient("proposal", CallHooks{})
|
||||
client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -403,150 +369,9 @@ func TestOperatorPromptOverridesUsePerCallSchemaAndOptionalSharedText(t *testing
|
||||
}
|
||||
}
|
||||
|
||||
func TestProfilePresenceBackendAuthorityAndProviderParametersReachWire(t *testing.T) {
|
||||
t.Setenv("CATALOG_TRAP_KEY", "catalog-secret-must-not-be-used")
|
||||
cases := []struct {
|
||||
name string
|
||||
parentTemperature string
|
||||
childTemperature string
|
||||
wantTemperature bool
|
||||
}{
|
||||
{name: "explicit zero", parentTemperature: " temperature: 0.8\n", childTemperature: " temperature: 0\n", wantTemperature: true},
|
||||
{name: "omitted", wantTemperature: false},
|
||||
{name: "null clears inherited control", parentTemperature: " temperature: 0.8\n", childTemperature: " temperature: null\n", wantTemperature: false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
profileDocument := "profiles:\n \"parent~/partial\":\n backend: openrouter\n model: parent-model\n" + tc.parentTemperature +
|
||||
" base:\n extends: \"parent~/partial\"\n model: full-model\n timeout_seconds: 0\n" + tc.childTemperature +
|
||||
" extra_params:\n routing:\n overridden: profile\n profile_only: retained\n cleared: null\n"
|
||||
operatorBackend := `backends:
|
||||
openrouter:
|
||||
endpoint: https://operator.invalid/v1
|
||||
concurrency_limit: 91
|
||||
queue_capacity: 92
|
||||
api_key_env: CATALOG_TRAP_KEY
|
||||
extra_params:
|
||||
routing:
|
||||
inherited: backend
|
||||
overridden: backend
|
||||
cleared: remove-me
|
||||
`
|
||||
documents := map[string]string{
|
||||
"unrelated-profile-file.yml": profileDocument,
|
||||
"provider-tuning.yaml": operatorBackend,
|
||||
}
|
||||
var wire map[string]json.RawMessage
|
||||
runtime, count, _ := adapterFixtureWithDocuments(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, func(r *http.Request) {
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer adapter-secret" {
|
||||
t.Errorf("application credential was not authoritative: %q", got)
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&wire); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}, "openrouter", documents)
|
||||
profile := runtime.Profiles["proposal"]
|
||||
if profile.EffectiveModelParams.APIKeyEnv != "" || profile.EffectiveModelParams.Endpoint == "https://operator.invalid/v1" || profile.EffectiveModelParams.TimeoutSeconds != 0 {
|
||||
t.Fatalf("application backend mapping or zero timeout lost: %+v", profile.EffectiveModelParams)
|
||||
}
|
||||
client, err := runtime.NewRoleClient("proposal", CallHooks{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var out map[string]any
|
||||
response, err := client.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &out)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if count.Load() != 1 {
|
||||
t.Fatalf("provider calls = %d, want 1", count.Load())
|
||||
}
|
||||
temperature, hasTemperature := wire["temperature"]
|
||||
if hasTemperature != tc.wantTemperature {
|
||||
t.Fatalf("temperature presence = %t, want %t: %s", hasTemperature, tc.wantTemperature, wire)
|
||||
}
|
||||
if tc.wantTemperature {
|
||||
var value float64
|
||||
if err := json.Unmarshal(temperature, &value); err != nil || value != 0 {
|
||||
t.Fatalf("explicit zero temperature = %s (%v)", temperature, err)
|
||||
}
|
||||
}
|
||||
var routing map[string]any
|
||||
if err := json.Unmarshal(wire["routing"], &routing); err != nil {
|
||||
t.Fatalf("provider routing fields missing: %s (%v)", wire, err)
|
||||
}
|
||||
if routing["inherited"] != "backend" || routing["overridden"] != "profile" || routing["profile_only"] != "retained" {
|
||||
t.Fatalf("recursively overlaid provider fields = %#v", routing)
|
||||
}
|
||||
if value, exists := routing["cleared"]; !exists || value != nil {
|
||||
t.Fatalf("null did not replace inherited provider field: %#v", routing)
|
||||
}
|
||||
if response.Model != "full-model" || response.Prepared == nil {
|
||||
t.Fatalf("execution did not use selected profile: %+v", response)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedAdapterUsesCapturedProfileAndPromptUntilReconstructed(t *testing.T) {
|
||||
var posted []struct {
|
||||
Model string `json:"model"`
|
||||
Messages []structuredoutput.Message `json:"messages"`
|
||||
}
|
||||
runtime, count, dir := adapterFixtureWithPrompt(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, func(r *http.Request) {
|
||||
var envelope struct {
|
||||
Model string `json:"model"`
|
||||
Messages []structuredoutput.Message `json:"messages"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&envelope); err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
posted = append(posted, envelope)
|
||||
}, operatorPromptDocument("Initial operator instructions.", "Initial payload."))
|
||||
client, err := runtime.NewRoleClient("proposal", CallHooks{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
profile := "profiles:\n base:\n backend: offline\n model: edited-model\n"
|
||||
if err := os.WriteFile(filepath.Join(dir, "profiles", "definitions.yaml"), []byte(profile), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
updatedPrompt := operatorPromptDocument("Edited operator instructions.", "Edited payload.")
|
||||
if err := os.WriteFile(filepath.Join(dir, "profiles", "operator-prompt.yaml"), []byte(updatedPrompt), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var out map[string]any
|
||||
response, err := client.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &out)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if response.Model != "full-model" || response.Prepared == nil || response.Prepared.Messages[0].Content != "Initial operator instructions." || posted[0].Model != "full-model" {
|
||||
t.Fatalf("existing runtime did not retain its captured configuration: response=%+v request=%+v", response, posted[0])
|
||||
}
|
||||
|
||||
main := filepath.Join(dir, "config.yml")
|
||||
newRuntime, err := NewRuntime(context.Background(), runtime.Config, main, http.DefaultClient)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
newClient, err := newRuntime.NewRoleClient("proposal", CallHooks{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
response, err = newClient.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &out)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if int(count.Load()) != 2 || response.Model != "edited-model" || response.Prepared == nil || response.Prepared.Messages[0].Content != "Edited operator instructions." || posted[1].Model != "edited-model" {
|
||||
t.Fatalf("new runtime did not capture edited configuration: response=%+v request=%+v", response, posted[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedAdapterRejectsTargetDriftAndPreparationErrors(t *testing.T) {
|
||||
runtime, count, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, nil)
|
||||
client, err := runtime.NewRoleClient("proposal", CallHooks{})
|
||||
client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -563,7 +388,7 @@ func TestPreparedAdapterCancellationStopsRetryWait(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
var once sync.Once
|
||||
client, err := runtime.NewRoleClient("proposal", CallHooks{OnEvent: func(e pk.ExecutionEvent) {
|
||||
client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{OnEvent: func(e pk.ExecutionEvent) {
|
||||
if e.Kind == pk.ExecutionAttempt && e.Dispatched {
|
||||
once.Do(cancel)
|
||||
}
|
||||
@@ -580,11 +405,9 @@ func TestPreparedAdapterCancellationStopsRetryWait(t *testing.T) {
|
||||
|
||||
func TestPreparedAdapterDistinguishesOperationTimeoutFromParentCancellation(t *testing.T) {
|
||||
blocking := func(*http.Request) { time.Sleep(500 * time.Millisecond) }
|
||||
runtime, _, _ := adapterFixtureWithDocuments(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, blocking, "offline", map[string]string{
|
||||
"definitions.yaml": "profiles:\n base:\n backend: offline\n model: full-model\n timeout_seconds: 0\n",
|
||||
})
|
||||
runtime, _, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, blocking)
|
||||
runtime.Config.LLM.OperationTimeout = 200 * time.Millisecond
|
||||
client, err := runtime.NewRoleClient("proposal", CallHooks{})
|
||||
client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -605,7 +428,7 @@ func TestPreparedAdapterDistinguishesOperationTimeoutFromParentCancellation(t *t
|
||||
|
||||
func TestPreparedAdapterClassifiesSetupAndDomainDecodeFailures(t *testing.T) {
|
||||
runtime, _, _ := adapterFixture(t, []reply{{200, `{"validations":[{"correction_index":9999999999999999999999999,"approved":true,"confidence":0.8,"reason":"ok"}]}`}}, 0, 0, nil)
|
||||
client, err := runtime.NewRoleClient("validation", CallHooks{})
|
||||
client, err := runtime.NewRoleClient("validation", "base", runtime.Config.LLM, CallHooks{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -638,7 +461,7 @@ func TestPreparedAdapterClassifiesSetupAndDomainDecodeFailures(t *testing.T) {
|
||||
|
||||
func TestPreparedAdapterNonRetryableProviderStatus(t *testing.T) {
|
||||
runtime, count, _ := adapterFixture(t, []reply{{http.StatusBadRequest, ""}}, 2, 0, nil)
|
||||
client, err := runtime.NewRoleClient("proposal", CallHooks{})
|
||||
client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user