eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Complete assembled validation and migrate remaining tests

Browse Source
This commit is contained in:
Eric Rakestraw
2026-11-10 18:13:12 +00:00
parent ba1d5d2367
commit 2844ce7e6a
44 changed files with 577 additions and 1624 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
internal/framework/llm/adapter.go
View File

@@ -100,6 +100,9 @@ func (c *RoleClient) CompleteStructured(parent context.Context, req structuredou
return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "unknown structured response schema", Cause: contractErr}
}
vars := prompts.WithSharedVariables(req.Variables)
for name, value := range prompts.SharedVariables() {
vars[name] = value
}
handle, prepErr := c.runtime.Engine.PrepareExecution(ctx, pk.RunRequest{
PromptID: req.PromptID, ProfileID: c.profileID, Vars: vars,
TransientRetries: &c.settings.TransientRetries,

215
internal/framework/llm/adapter_capacity_test.go
View File

@@ -2,216 +2,121 @@ package llm
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
pk "gitea.maximumdirect.net/eric/promptkit"
ck "go.maximumdirect.net/configkit"
)
func TestRolesShareBackendCapacityAndReleaseAfterCancellation(t *testing.T) {
t.Setenv("INHERITED_TEST_KEY", "synthetic-inherited-runtime-secret")
t.Setenv("OPERATOR_BACKEND_KEY", "synthetic-operator-backend-secret")
entered := make(chan struct{}, 3)
release := make(chan struct{})
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if got := r.Header.Get("Authorization"); got != "Bearer synthetic-inherited-runtime-secret" {
t.Errorf("inherited credential header: %q", got)
}
entered <- struct{}{}
<-release
content, _ := json.Marshal(`{"corrections":[]}`)
_, _ = fmt.Fprintf(w, `{"choices":[{"message":{"content":%s}}]}`, content)
}))
defer server.Close()
func capacityFixture(t *testing.T, resources string, transport http.RoundTripper) *Runtime {
t.Helper()
dir := t.TempDir()
if err := os.Mkdir(filepath.Join(dir, "profiles"), 0700); err != nil {
t.Fatal(err)
}
main := filepath.Join(dir, "config.yml")
body := fmt.Sprintf("version: 2\nbackends:\n shared-base:\n endpoint: %s/v1\n credential: INHERITED_TEST_KEY\n concurrency: 1\n queue_capacity: 0\n shared:\n extends: shared-base\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n repair_attempts: 0\n", server.URL)
if err := os.WriteFile(main, []byte(body), 0600); err != nil {
t.Fatal(err)
}
for _, name := range []string{"proposal", "validation"} {
profile := "profiles:\n " + name + ":\n backend: shared\n model: local\n"
if err := os.WriteFile(filepath.Join(dir, "profiles", name+".yaml"), []byte(profile), 0600); err != nil {
for name, content := range map[string]string{"config.yml": "version: 3\n", "prompts.yml": resources} {
if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0600); err != nil {
t.Fatal(err)
}
}
operatorBackend := `backends:
shared:
endpoint: https://operator.invalid/v1
concurrency_limit: 77
queue_capacity: 88
api_key_env: OPERATOR_BACKEND_KEY
`
if err := os.WriteFile(filepath.Join(dir, "profiles", "backend-tuning.yaml"), []byte(operatorBackend), 0600); err != nil {
t.Fatal(err)
}
loaded, err := config.Load(ck.File(main), "")
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
if err != nil {
t.Fatal(err)
}
runtime, err := NewRuntime(context.Background(), loaded.Config, main, server.Client())
runtime, err := NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
if err != nil {
t.Fatal(err)
}
proposal, err := runtime.NewRoleClient("proposal", CallHooks{})
return runtime
}
func capacityResponse(r *http.Request) *http.Response {
return &http.Response{StatusCode: 200, Header: make(http.Header), Request: r, Body: io.NopCloser(strings.NewReader(`{"choices":[{"message":{"content":"{\"corrections\":[]}"}}]}`))}
}
func TestRolesShareBackendCapacityAndReleaseAfterCancellation(t *testing.T) {
entered := make(chan struct{}, 3)
release := make(chan struct{})
transport := transportFunc(func(r *http.Request) (*http.Response, error) {
entered <- struct{}{}
select {
case <-release:
return capacityResponse(r), nil
case <-r.Context().Done():
return nil, r.Context().Err()
}
})
resources := "backends:\n - id: shared\n endpoint: https://example.invalid/v1\n concurrency_limit: 1\n queue_capacity: 1\nprofiles:\n - id: proposal\n backend: shared\n model: local\n - id: validation\n backend: shared\n model: local\n"
runtime := capacityFixture(t, resources, transport)
settings := config.LLM{OperationTimeout: 3 * time.Second}
proposal, err := runtime.NewRoleClient("proposal", "proposal", settings, CallHooks{})
if err != nil {
t.Fatal(err)
}
validation, err := runtime.NewRoleClient("validation", CallHooks{})
validation, err := runtime.NewRoleClient("validation", "validation", settings, CallHooks{})
if err != nil {
t.Fatal(err)
}
request := structuredoutput.Request{PromptID: prompts.PromptIDModuleGlossaryProposal, PromptVersion: prompts.VersionV1, Variables: map[string]string{"TranscriptDescriptionBlock": "", "GlossaryJSON": "[]", "SectionJSON": "[]"}, Schema: responseschema.CorrectionSetKey}
firstDone := make(chan error, 1)
request := adapterRequest(responseschema.CorrectionSetKey)
done := make(chan error, 1)
go func() {
var output map[string]any
_, callErr := proposal.CompleteStructured(context.Background(), request, &output)
firstDone <- callErr
_, err := proposal.CompleteStructured(context.Background(), request, &map[string]any{})
done <- err
}()
select {
case <-entered:
case <-time.After(2 * time.Second):
t.Fatal("first call did not enter provider")
}
var output map[string]any
capacityResponse, err := validation.CompleteStructured(context.Background(), request, &output)
if !errors.Is(err, pk.ErrCapacityExceeded) {
t.Fatalf("other role did not share capacity: %v", err)
}
var capacityFailure *structuredoutput.Failure
if !errors.As(err, &capacityFailure) || capacityFailure.Class != structuredoutput.FailureRecoverable || capacityFailure.Reason != outcomes.ReasonCapacityExceeded || !capacityResponse.Execution.Available || capacityResponse.Execution.ProviderAttempts != 0 {
t.Fatalf("capacity outcome: failure=%#v counts=%+v", capacityFailure, capacityResponse.Execution)
}
select {
case <-entered:
t.Fatal("second role reached provider despite full backend")
default:
}
close(release)
if err := <-firstDone; err != nil {
t.Fatal(err)
}
_, err = validation.CompleteStructured(context.Background(), request, &output)
if err != nil {
t.Fatalf("capacity was not released: %v", err)
}
for len(entered) > 0 {
<-entered
}
release = make(chan struct{})
if err := os.WriteFile(main, []byte(strings.Replace(body, "queue_capacity: 0", "queue_capacity: 1", 1)), 0600); err != nil {
t.Fatal(err)
}
loaded, err = config.Load(ck.File(main), "")
if err != nil {
t.Fatal(err)
}
runtime, err = NewRuntime(context.Background(), loaded.Config, main, server.Client())
if err != nil {
t.Fatal(err)
}
proposal, err = runtime.NewRoleClient("proposal", CallHooks{})
if err != nil {
t.Fatal(err)
}
validation, err = runtime.NewRoleClient("validation", CallHooks{})
if err != nil {
t.Fatal(err)
}
go func() {
var out map[string]any
_, callErr := proposal.CompleteStructured(context.Background(), request, &out)
firstDone <- callErr
}()
select {
case <-entered:
case <-time.After(2 * time.Second):
t.Fatal("first queued-capacity call did not dispatch")
}
queuedCtx, cancelQueued := context.WithTimeout(context.Background(), 30*time.Millisecond)
defer cancelQueued()
_, err = validation.CompleteStructured(queuedCtx, request, &output)
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond)
defer cancel()
_, err = validation.CompleteStructured(ctx, request, &map[string]any{})
if !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("queued call did not honor deadline: %v", err)
}
if !errors.As(err, &capacityFailure) || capacityFailure.Class != structuredoutput.FailureCanceled || capacityFailure.Reason != outcomes.ReasonCanceled {
t.Fatalf("queued parent deadline classification: %#v", capacityFailure)
select {
case <-entered:
t.Fatal("queued call dispatched")
default:
}
close(release)
if err := <-firstDone; err != nil {
if err := <-done; err != nil {
t.Fatal(err)
}
_, err = validation.CompleteStructured(context.Background(), request, &output)
if err != nil {
t.Fatalf("queued cancellation leaked capacity: %v", err)
if _, err := validation.CompleteStructured(context.Background(), request, &map[string]any{}); err != nil {
t.Fatalf("cancellation leaked capacity: %v", err)
}
}
func TestDistinctBackendsHaveIndependentCapacity(t *testing.T) {
entered := make(chan struct{}, 2)
release := make(chan struct{})
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
transport := transportFunc(func(r *http.Request) (*http.Response, error) {
entered <- struct{}{}
<-release
schema := map[string]any{}
_ = json.NewDecoder(r.Body).Decode(&schema)
content, _ := json.Marshal(`{"corrections":[]}`)
_, _ = fmt.Fprintf(w, `{"choices":[{"message":{"content":%s}}]}`, content)
}))
defer server.Close()
dir := t.TempDir()
if err := os.Mkdir(filepath.Join(dir, "profiles"), 0700); err != nil {
t.Fatal(err)
}
main := filepath.Join(dir, "config.yml")
body := fmt.Sprintf("version: 2\nbackends:\n first:\n endpoint: %s/v1\n concurrency: 1\n queue_capacity: 0\n second:\n endpoint: %s/v1\n concurrency: 1\n queue_capacity: 0\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n repair_attempts: 0\n", server.URL, server.URL)
if err := os.WriteFile(main, []byte(body), 0600); err != nil {
t.Fatal(err)
}
for name, backend := range map[string]string{"proposal": "first", "validation": "second"} {
if err := os.WriteFile(filepath.Join(dir, "profiles", name+".yaml"), []byte("profiles:\n "+name+":\n backend: "+backend+"\n model: local\n"), 0600); err != nil {
t.Fatal(err)
}
}
loaded, err := config.Load(ck.File(main), "")
return capacityResponse(r), nil
})
resources := "backends:\n - id: first\n endpoint: https://example.invalid/v1\n concurrency_limit: 1\n queue_capacity: 0\n - id: second\n endpoint: https://example.invalid/v1\n concurrency_limit: 1\n queue_capacity: 0\nprofiles:\n - id: proposal\n backend: first\n model: local\n - id: validation\n backend: second\n model: local\n"
runtime := capacityFixture(t, resources, transport)
settings := config.LLM{OperationTimeout: 3 * time.Second}
first, err := runtime.NewRoleClient("proposal", "proposal", settings, CallHooks{})
if err != nil {
t.Fatal(err)
}
runtime, err := NewRuntime(context.Background(), loaded.Config, main, server.Client())
second, err := runtime.NewRoleClient("validation", "validation", settings, CallHooks{})
if err != nil {
t.Fatal(err)
}
proposal, err := runtime.NewRoleClient("proposal", CallHooks{})
if err != nil {
t.Fatal(err)
}
validation, err := runtime.NewRoleClient("validation", CallHooks{})
if err != nil {
t.Fatal(err)
}
request := structuredoutput.Request{PromptID: prompts.PromptIDModuleGlossaryProposal, PromptVersion: prompts.VersionV1, Variables: map[string]string{"TranscriptDescriptionBlock": "", "GlossaryJSON": "[]", "SectionJSON": "[]"}, Schema: responseschema.CorrectionSetKey}
request := adapterRequest(responseschema.CorrectionSetKey)
done := make(chan error, 2)
go func() {
var out map[string]any
_, e := proposal.CompleteStructured(context.Background(), request, &out)
done <- e
_, err := first.CompleteStructured(context.Background(), request, &map[string]any{})
done <- err
}()
select {
case <-entered:
@@ -219,13 +124,13 @@ func TestDistinctBackendsHaveIndependentCapacity(t *testing.T) {
t.Fatal("first backend did not dispatch")
}
go func() {
var out map[string]any
_, e := validation.CompleteStructured(context.Background(), request, &out)
done <- e
_, err := second.CompleteStructured(context.Background(), request, &map[string]any{})
done <- err
}()
select {
case <-entered:
case <-time.After(2 * time.Second):
close(release)
t.Fatal("second backend was blocked by first")
}
close(release)

267
internal/framework/llm/adapter_test.go
View File

@@ -5,8 +5,8 @@ import (
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
@@ -15,13 +15,13 @@ import (
"testing"
"time"
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
pk "gitea.maximumdirect.net/eric/promptkit"
ck "go.maximumdirect.net/configkit"
pk "go.maximumdirect.net/promptkit"
)
type reply struct {
@@ -30,85 +30,56 @@ type reply struct {
}
func adapterFixture(t *testing.T, replies []reply, retries, repairs int, hook func(*http.Request)) (*Runtime, *atomic.Int32, string) {
return adapterFixtureWithDocuments(t, replies, retries, repairs, hook, "offline", map[string]string{
"definitions.yaml": "profiles:\n base:\n backend: offline\n model: full-model\n",
})
return adapterFixtureWithPrompt(t, replies, retries, repairs, hook, "")
}
func adapterFixtureWithPrompt(t *testing.T, replies []reply, retries, repairs int, hook func(*http.Request), operatorPrompt string) (*Runtime, *atomic.Int32, string) {
documents := map[string]string{
"definitions.yaml": "profiles:\n base:\n backend: offline\n model: full-model\n",
}
if operatorPrompt != "" {
documents["operator-prompt.yaml"] = operatorPrompt
}
return adapterFixtureWithDocuments(t, replies, retries, repairs, hook, "offline", documents)
}
func adapterFixtureWithDocuments(t *testing.T, replies []reply, retries, repairs int, hook func(*http.Request), backendID string, documents map[string]string) (*Runtime, *atomic.Int32, string) {
t.Helper()
var count atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
t.Setenv("ADAPTER_KEY", "adapter-secret")
dir := t.TempDir()
resources := "backends:\n - id: offline\n endpoint: https://provider.invalid/v1\n api_key_env: ADAPTER_KEY\n concurrency_limit: 1\n queue_capacity: 0\nprofiles:\n - id: base\n backend: offline\n model: full-model\n"
for name, body := range map[string]string{"config.yml": "version: 3\n", "prompts.yml": resources + operatorPrompt} {
if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0600); err != nil {
t.Fatal(err)
}
}
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
if err != nil {
t.Fatal(err)
}
transport := transportFunc(func(r *http.Request) (*http.Response, error) {
call := int(count.Add(1))
if hook != nil {
hook(r)
}
step := replies[len(replies)-1]
if call <= len(replies) {
step = replies[call-1]
if err := r.Context().Err(); err != nil {
return nil, err
}
step := replies[min(call-1, len(replies)-1)]
if step.status == 0 {
connection, _, hijackErr := w.(http.Hijacker).Hijack()
if hijackErr != nil {
t.Error(hijackErr)
return
return nil, errors.New("synthetic transport error")
}
body := `{"error":{"message":"provider error"}}`
if step.status == http.StatusOK {
body = `{"choices":[]}`
if step.content != "MALFORMED_ENVELOPE" {
encoded, _ := json.Marshal(step.content)
body = fmt.Sprintf(`{"model":"returned-model","choices":[{"message":{"content":%s}}],"usage":{"prompt_tokens":2,"completion_tokens":3,"total_tokens":5}}`, encoded)
}
_ = connection.Close()
return
}
if step.status != http.StatusOK {
w.WriteHeader(step.status)
_, _ = fmt.Fprint(w, `{"error":{"message":"provider error"}}`)
return
}
if step.content == "MALFORMED_ENVELOPE" {
_, _ = fmt.Fprint(w, `{"choices":[]}`)
return
}
encoded, _ := json.Marshal(step.content)
w.Header().Set("Content-Type", "application/json")
_, _ = fmt.Fprintf(w, `{"model":"returned-model","choices":[{"message":{"content":%s}}],"usage":{"prompt_tokens":2,"completion_tokens":3,"total_tokens":5}}`, encoded)
}))
t.Cleanup(server.Close)
t.Setenv("ADAPTER_KEY", "adapter-secret")
dir := t.TempDir()
profileDirectory := filepath.Join(dir, "profiles")
if err := os.Mkdir(profileDirectory, 0700); err != nil {
t.Fatal(err)
}
base := fmt.Sprintf("version: 2\nprofiles_dir: profiles\nbackends:\n %s:\n endpoint: %s/v1\n credential: ADAPTER_KEY\n concurrency: 1\n queue_capacity: 0\nllm:\n proposal_profile: base\n transient_retries: %d\n repair_attempts: %d\n", backendID, server.URL, retries, repairs)
main := filepath.Join(dir, "config.yml")
if err := os.WriteFile(main, []byte(base), 0600); err != nil {
t.Fatal(err)
}
for name, content := range documents {
if err := os.WriteFile(filepath.Join(profileDirectory, name), []byte(content), 0600); err != nil {
t.Fatal(err)
}
}
loaded, err := config.Load(ck.File(main), "")
if err != nil {
t.Fatal(err)
}
runtime, err := NewRuntime(context.Background(), loaded.Config, main, server.Client())
return &http.Response{StatusCode: step.status, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
})
runtime, err := NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
if err != nil {
t.Fatal(err)
}
runtime.Config.LLM = config.LLM{OperationTimeout: 10 * time.Second, TransientRetries: retries, RepairAttempts: repairs}
return runtime, &count, dir
}
func operatorPromptDocument(system, user string) string {
return fmt.Sprintf("prompts:\n %s:\n v1:\n messages:\n - role: system\n content: %q\n - role: user\n content: %q\n output:\n format: text\n validation_mode: basic\n", prompts.PromptIDModuleGlossaryProposal, system, user)
return fmt.Sprintf("collections:\n audita:\n prompts:\n - id: modules.glossary.proposal\n messages:\n - role: system\n content: %q\n - role: user\n content: %q\n output:\n format: text\n validation_mode: none\n", system, user)
}
func adapterRequest(schema responseschema.Key) structuredoutput.Request {
@@ -118,7 +89,7 @@ func adapterRequest(schema responseschema.Key) structuredoutput.Request {
id = prompts.PromptIDValidatorEditorialReview
vars = map[string]string{"TranscriptDescriptionBlock": "", "PayloadJSON": "[]"}
}
return structuredoutput.Request{PromptID: id, PromptVersion: prompts.VersionV1, Variables: vars, Schema: schema, Context: structuredoutput.DiagnosticContext{ModuleKey: "glossary"}}
return structuredoutput.Request{PromptID: id, Variables: vars, Schema: schema, Context: structuredoutput.DiagnosticContext{ModuleKey: "glossary"}}
}
func TestPreparedAdapterRepairsAndClassifies(t *testing.T) {
@@ -154,7 +125,7 @@ func TestPreparedAdapterRepairsAndClassifies(t *testing.T) {
}
var events []pk.ExecutionEvent
var records []structuredoutput.CallRecord
client, err := runtime.NewRoleClient(role, CallHooks{OnEvent: func(e pk.ExecutionEvent) { events = append(events, e) }, OnCall: func(r structuredoutput.CallRecord) { records = append(records, r) }})
client, err := runtime.NewRoleClient(role, "base", runtime.Config.LLM, CallHooks{OnEvent: func(e pk.ExecutionEvent) { events = append(events, e) }, OnCall: func(r structuredoutput.CallRecord) { records = append(records, r) }})
if err != nil {
t.Fatal(err)
}
@@ -234,7 +205,7 @@ func TestCorrectionSchemaReplacementValuesThroughAdapter(t *testing.T) {
}
payload := `{"corrections":[{` + strings.Join(fields, ",") + `}]}`
runtime, count, _ := adapterFixture(t, []reply{{200, payload}}, 0, tc.repairs, nil)
client, err := runtime.NewRoleClient("proposal", CallHooks{})
client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{})
if err != nil {
t.Fatal(err)
}
@@ -280,7 +251,7 @@ func TestPreparedAdapterMetadataAndTargetGuard(t *testing.T) {
t.Error(err)
}
})
client, err := runtime.NewRoleClient("proposal", CallHooks{})
client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{})
if err != nil {
t.Fatal(err)
}
@@ -314,12 +285,7 @@ func TestPreparedAdapterMetadataAndTargetGuard(t *testing.T) {
if !found {
t.Fatal("hardening policy missing")
}
backend := runtime.Config.Backends["offline"]
backend.Endpoint = "http://127.0.0.1:1/v1"
runtime.Config.Backends["offline"] = backend
if _, err := client.CompleteStructured(context.Background(), req, &out); err == nil || !strings.Contains(err.Error(), "target changed") || count.Load() != 1 {
t.Fatalf("changed application backend target dispatched: %v, %d", err, count.Load())
}
}
func TestOperatorPromptOverridesUsePerCallSchemaAndOptionalSharedText(t *testing.T) {
@@ -365,7 +331,7 @@ func TestOperatorPromptOverridesUsePerCallSchemaAndOptionalSharedText(t *testing
}
posted = append(posted, envelope)
}, operatorPromptDocument(tc.system, tc.user))
client, err := runtime.NewRoleClient("proposal", CallHooks{})
client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{})
if err != nil {
t.Fatal(err)
}
@@ -403,150 +369,9 @@ func TestOperatorPromptOverridesUsePerCallSchemaAndOptionalSharedText(t *testing
}
}
func TestProfilePresenceBackendAuthorityAndProviderParametersReachWire(t *testing.T) {
t.Setenv("CATALOG_TRAP_KEY", "catalog-secret-must-not-be-used")
cases := []struct {
name string
parentTemperature string
childTemperature string
wantTemperature bool
}{
{name: "explicit zero", parentTemperature: " temperature: 0.8\n", childTemperature: " temperature: 0\n", wantTemperature: true},
{name: "omitted", wantTemperature: false},
{name: "null clears inherited control", parentTemperature: " temperature: 0.8\n", childTemperature: " temperature: null\n", wantTemperature: false},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
profileDocument := "profiles:\n \"parent~/partial\":\n backend: openrouter\n model: parent-model\n" + tc.parentTemperature +
" base:\n extends: \"parent~/partial\"\n model: full-model\n timeout_seconds: 0\n" + tc.childTemperature +
" extra_params:\n routing:\n overridden: profile\n profile_only: retained\n cleared: null\n"
operatorBackend := `backends:
openrouter:
endpoint: https://operator.invalid/v1
concurrency_limit: 91
queue_capacity: 92
api_key_env: CATALOG_TRAP_KEY
extra_params:
routing:
inherited: backend
overridden: backend
cleared: remove-me
`
documents := map[string]string{
"unrelated-profile-file.yml": profileDocument,
"provider-tuning.yaml": operatorBackend,
}
var wire map[string]json.RawMessage
runtime, count, _ := adapterFixtureWithDocuments(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, func(r *http.Request) {
if got := r.Header.Get("Authorization"); got != "Bearer adapter-secret" {
t.Errorf("application credential was not authoritative: %q", got)
}
if err := json.NewDecoder(r.Body).Decode(&wire); err != nil {
t.Error(err)
}
}, "openrouter", documents)
profile := runtime.Profiles["proposal"]
if profile.EffectiveModelParams.APIKeyEnv != "" || profile.EffectiveModelParams.Endpoint == "https://operator.invalid/v1" || profile.EffectiveModelParams.TimeoutSeconds != 0 {
t.Fatalf("application backend mapping or zero timeout lost: %+v", profile.EffectiveModelParams)
}
client, err := runtime.NewRoleClient("proposal", CallHooks{})
if err != nil {
t.Fatal(err)
}
var out map[string]any
response, err := client.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &out)
if err != nil {
t.Fatal(err)
}
if count.Load() != 1 {
t.Fatalf("provider calls = %d, want 1", count.Load())
}
temperature, hasTemperature := wire["temperature"]
if hasTemperature != tc.wantTemperature {
t.Fatalf("temperature presence = %t, want %t: %s", hasTemperature, tc.wantTemperature, wire)
}
if tc.wantTemperature {
var value float64
if err := json.Unmarshal(temperature, &value); err != nil || value != 0 {
t.Fatalf("explicit zero temperature = %s (%v)", temperature, err)
}
}
var routing map[string]any
if err := json.Unmarshal(wire["routing"], &routing); err != nil {
t.Fatalf("provider routing fields missing: %s (%v)", wire, err)
}
if routing["inherited"] != "backend" || routing["overridden"] != "profile" || routing["profile_only"] != "retained" {
t.Fatalf("recursively overlaid provider fields = %#v", routing)
}
if value, exists := routing["cleared"]; !exists || value != nil {
t.Fatalf("null did not replace inherited provider field: %#v", routing)
}
if response.Model != "full-model" || response.Prepared == nil {
t.Fatalf("execution did not use selected profile: %+v", response)
}
})
}
}
func TestPreparedAdapterUsesCapturedProfileAndPromptUntilReconstructed(t *testing.T) {
var posted []struct {
Model string `json:"model"`
Messages []structuredoutput.Message `json:"messages"`
}
runtime, count, dir := adapterFixtureWithPrompt(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, func(r *http.Request) {
var envelope struct {
Model string `json:"model"`
Messages []structuredoutput.Message `json:"messages"`
}
if err := json.NewDecoder(r.Body).Decode(&envelope); err != nil {
t.Error(err)
return
}
posted = append(posted, envelope)
}, operatorPromptDocument("Initial operator instructions.", "Initial payload."))
client, err := runtime.NewRoleClient("proposal", CallHooks{})
if err != nil {
t.Fatal(err)
}
profile := "profiles:\n base:\n backend: offline\n model: edited-model\n"
if err := os.WriteFile(filepath.Join(dir, "profiles", "definitions.yaml"), []byte(profile), 0600); err != nil {
t.Fatal(err)
}
updatedPrompt := operatorPromptDocument("Edited operator instructions.", "Edited payload.")
if err := os.WriteFile(filepath.Join(dir, "profiles", "operator-prompt.yaml"), []byte(updatedPrompt), 0600); err != nil {
t.Fatal(err)
}
var out map[string]any
response, err := client.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &out)
if err != nil {
t.Fatal(err)
}
if response.Model != "full-model" || response.Prepared == nil || response.Prepared.Messages[0].Content != "Initial operator instructions." || posted[0].Model != "full-model" {
t.Fatalf("existing runtime did not retain its captured configuration: response=%+v request=%+v", response, posted[0])
}
main := filepath.Join(dir, "config.yml")
newRuntime, err := NewRuntime(context.Background(), runtime.Config, main, http.DefaultClient)
if err != nil {
t.Fatal(err)
}
newClient, err := newRuntime.NewRoleClient("proposal", CallHooks{})
if err != nil {
t.Fatal(err)
}
response, err = newClient.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &out)
if err != nil {
t.Fatal(err)
}
if int(count.Load()) != 2 || response.Model != "edited-model" || response.Prepared == nil || response.Prepared.Messages[0].Content != "Edited operator instructions." || posted[1].Model != "edited-model" {
t.Fatalf("new runtime did not capture edited configuration: response=%+v request=%+v", response, posted[1])
}
}
func TestPreparedAdapterRejectsTargetDriftAndPreparationErrors(t *testing.T) {
runtime, count, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, nil)
client, err := runtime.NewRoleClient("proposal", CallHooks{})
client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{})
if err != nil {
t.Fatal(err)
}
@@ -563,7 +388,7 @@ func TestPreparedAdapterCancellationStopsRetryWait(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
var once sync.Once
client, err := runtime.NewRoleClient("proposal", CallHooks{OnEvent: func(e pk.ExecutionEvent) {
client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{OnEvent: func(e pk.ExecutionEvent) {
if e.Kind == pk.ExecutionAttempt && e.Dispatched {
once.Do(cancel)
}
@@ -580,11 +405,9 @@ func TestPreparedAdapterCancellationStopsRetryWait(t *testing.T) {
func TestPreparedAdapterDistinguishesOperationTimeoutFromParentCancellation(t *testing.T) {
blocking := func(*http.Request) { time.Sleep(500 * time.Millisecond) }
runtime, _, _ := adapterFixtureWithDocuments(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, blocking, "offline", map[string]string{
"definitions.yaml": "profiles:\n base:\n backend: offline\n model: full-model\n timeout_seconds: 0\n",
})
runtime, _, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, blocking)
runtime.Config.LLM.OperationTimeout = 200 * time.Millisecond
client, err := runtime.NewRoleClient("proposal", CallHooks{})
client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{})
if err != nil {
t.Fatal(err)
}
@@ -605,7 +428,7 @@ func TestPreparedAdapterDistinguishesOperationTimeoutFromParentCancellation(t *t
func TestPreparedAdapterClassifiesSetupAndDomainDecodeFailures(t *testing.T) {
runtime, _, _ := adapterFixture(t, []reply{{200, `{"validations":[{"correction_index":9999999999999999999999999,"approved":true,"confidence":0.8,"reason":"ok"}]}`}}, 0, 0, nil)
client, err := runtime.NewRoleClient("validation", CallHooks{})
client, err := runtime.NewRoleClient("validation", "base", runtime.Config.LLM, CallHooks{})
if err != nil {
t.Fatal(err)
}
@@ -638,7 +461,7 @@ func TestPreparedAdapterClassifiesSetupAndDomainDecodeFailures(t *testing.T) {
func TestPreparedAdapterNonRetryableProviderStatus(t *testing.T) {
runtime, count, _ := adapterFixture(t, []reply{{http.StatusBadRequest, ""}}, 2, 0, nil)
client, err := runtime.NewRoleClient("proposal", CallHooks{})
client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{})
if err != nil {
t.Fatal(err)
}

4
internal/framework/llm/observations_test.go
View File

@@ -10,7 +10,7 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
pk "gitea.maximumdirect.net/eric/promptkit"
pk "go.maximumdirect.net/promptkit"
)
func TestObservationSummarySeparatesLogicalCallsAttemptsAndUsage(t *testing.T) {
@@ -93,7 +93,7 @@ func TestObservationCollectorConcurrentAttributionAndSnapshotCopy(t *testing.T)
func TestRoleClientFeedsCollectorAcrossPreparationFailure(t *testing.T) {
runtime, count, _ := adapterFixture(t, []reply{{status: 200, content: `{"corrections":[]}`}}, 0, 0, nil)
collector := NewObservationCollector()
client, err := runtime.NewRoleClient("proposal", collector.Hooks())
client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, collector.Hooks())
if err != nil {
t.Fatal(err)
}

225
internal/framework/llm/runtime_assembly_test.go
View File

@@ -4,43 +4,17 @@ import (
"context"
"errors"
"io/fs"
"net/http"
"os"
"path"
"path/filepath"
"strings"
"testing"
"testing/fstest"
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
ck "go.maximumdirect.net/configkit"
)
func runtimeAssemblyFixture(t *testing.T, selections string, documents map[string]string) (config.Config, string) {
t.Helper()
dir := t.TempDir()
profileDir := filepath.Join(dir, "profiles")
if err := os.Mkdir(profileDir, 0700); err != nil {
t.Fatal(err)
}
main := filepath.Join(dir, "config.yml")
text := "version: 2\nprofiles_dir: profiles\nbackends:\n offline:\n endpoint: http://127.0.0.1:1/v1\n concurrency: 1\n queue_capacity: 0\nllm:\n" + selections
if err := os.WriteFile(main, []byte(text), 0600); err != nil {
t.Fatal(err)
}
for name, content := range documents {
if err := os.WriteFile(filepath.Join(profileDir, name), []byte(content), 0600); err != nil {
t.Fatal(err)
}
}
loaded, err := config.Load(ck.File(main), "")
if err != nil {
t.Fatal(err)
}
return loaded.Config, main
}
func copyEmbeddedPromptFilesystem(t *testing.T) fstest.MapFS {
t.Helper()
source := prompts.Filesystem()
@@ -62,166 +36,18 @@ func copyEmbeddedPromptFilesystem(t *testing.T) fstest.MapFS {
return files
}
func TestNewRuntimeRequiresCompleteSelectedProfiles(t *testing.T) {
validProposal := " backend: offline\n model: proposal-model\n"
cases := []struct {
name string
selections string
profiles string
role string
field string
}{
{
name: "omitted proposal model with derived validation selection",
selections: " proposal_profile: proposal\n",
profiles: "profiles:\n proposal:\n backend: offline\n",
role: "proposal",
field: "model",
},
{
name: "empty validation model",
selections: " proposal_profile: proposal\n validation_profile: validation\n",
profiles: "profiles:\n proposal:\n" + validProposal + " validation:\n backend: offline\n model: \"\"\n",
role: "validation",
field: "model",
},
{
name: "null proposal model",
selections: " proposal_profile: proposal\n",
profiles: "profiles:\n proposal:\n backend: offline\n model: null\n",
role: "proposal",
field: "model",
},
{
name: "cleared validation endpoint",
selections: " proposal_profile: proposal\n validation_profile: validation\n",
profiles: "profiles:\n proposal:\n" + validProposal + " validation:\n backend: offline\n endpoint: null\n model: validation-model\n",
role: "validation",
field: "endpoint",
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
cfg, main := runtimeAssemblyFixture(t, tc.selections, map[string]string{"definitions.yml": tc.profiles})
_, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}})
if err == nil || !strings.Contains(err.Error(), tc.role) || !strings.Contains(err.Error(), tc.field) {
t.Fatalf("selected incomplete profile was not rejected for %s.%s: %v", tc.role, tc.field, err)
}
})
}
}
func TestNewRuntimeAllowsUnselectedPartialProfiles(t *testing.T) {
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{
"definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n unfinished:\n backend: offline\n",
})
runtime, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}})
if err != nil {
t.Fatal(err)
}
if runtime.Profiles["proposal"].EffectiveModelParams.Model != "proposal-model" || runtime.Profiles["validation"].EffectiveModelParams.Model != "proposal-model" {
t.Fatalf("role profiles were not resolved: %+v", runtime.Profiles)
}
}
func TestNativeProfileInheritancePreservesControlsAndMergesProviderValues(t *testing.T) {
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: \"child profile\"\n", map[string]string{
"unrelated-name.yaml": `profiles:
"parent~/shared":
backend: offline
temperature: 0.4
top_p: 0.7
extra_params:
routing:
retained: from-parent
overridden: parent
cleared: remove-me
"child profile":
extends: "parent~/shared"
model: child-model
extra_params:
routing:
overridden: child
child_only: present
cleared: null
`,
})
runtime, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{}})
if err != nil {
t.Fatal(err)
}
profile := runtime.Profiles["proposal"]
target := profile.EffectiveModelParams
if profile.ProfileID != "child profile" || target.Model != "child-model" || target.Temperature != 0.4 || target.TopP != 0.7 {
t.Fatalf("inherited profile target = %+v", target)
}
routing, ok := target.ExtraParams["routing"].(map[string]any)
if !ok || routing["retained"] != "from-parent" || routing["overridden"] != "child" || routing["child_only"] != "present" {
t.Fatalf("recursively inherited provider values = %#v", target.ExtraParams)
}
if value, exists := routing["cleared"]; !exists || value != nil {
t.Fatalf("null did not replace the inherited provider value: %#v", routing)
}
}
func TestMissingSelectedProfileDirectoryIsOperatorInputError(t *testing.T) {
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{
"definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n",
})
if err := os.RemoveAll(filepath.Join(filepath.Dir(main), cfg.ProfilesDir)); err != nil {
t.Fatal(err)
}
_, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}})
if err == nil || IsAssetError(err) {
t.Fatalf("missing operator profile directory classification: %v", err)
}
}
func TestNewRuntimeRejectsMalformedUnselectedProfilesAndConflicts(t *testing.T) {
cases := []struct {
name string
document map[string]string
selection string
}{
{
name: "malformed unselected definition",
document: map[string]string{"definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n unused:\n temperature: 9\n"},
selection: " proposal_profile: proposal\n",
},
{
name: "undefined selected backend",
document: map[string]string{"definitions.yml": "profiles:\n proposal:\n backend: missing\n model: proposal-model\n"},
selection: " proposal_profile: proposal\n",
},
{
name: "conflicting directory contributions",
document: map[string]string{
"a.yml": "profiles:\n proposal:\n backend: offline\n model: first-model\n",
"b.yml": "profiles:\n proposal:\n model: second-model\n",
},
selection: " proposal_profile: proposal\n",
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
cfg, main := runtimeAssemblyFixture(t, tc.selection, tc.document)
_, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}})
if err == nil || IsAssetError(err) {
t.Fatalf("operator definition classification: %v", err)
}
})
}
}
func TestRuntimeClassifiesEmbeddedAssetsAndHonorsCancellation(t *testing.T) {
_, err := newRuntimeWithPromptFS(context.Background(), config.Config{}, "config.yml", nil, fstest.MapFS{})
loaded, err := config.Load(ck.Input{Source: ck.NoSource()}, "")
if err != nil {
t.Fatal(err)
}
_, err = newRuntimeWithPromptFS(context.Background(), loaded, nil, fstest.MapFS{})
if err == nil || !IsAssetError(err) || !errors.Is(err, fs.ErrNotExist) {
t.Fatalf("embedded discovery error was not preserved as an asset error: %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
_, err = newRuntimeWithPromptFS(ctx, config.Config{}, "config.yml", nil, fstest.MapFS{})
_, err = newRuntimeWithPromptFS(ctx, loaded, nil, fstest.MapFS{})
if !errors.Is(err, context.Canceled) || IsAssetError(err) {
t.Fatalf("caller cancellation classification: %v", err)
}
@@ -229,12 +55,13 @@ func TestRuntimeClassifiesEmbeddedAssetsAndHonorsCancellation(t *testing.T) {
func TestBrokenEmbeddedPromptFailsEmbeddedOnlyConstruction(t *testing.T) {
promptFS := copyEmbeddedPromptFilesystem(t)
assetPath := path.Join(prompts.AssetRoot, prompts.PromptIDModuleGrammarProposal+".yaml")
promptFS[assetPath] = &fstest.MapFile{Data: []byte("prompts:\n modules.grammar.proposal:\n v1:\n messages: []\n output:\n format: json\n validation_mode: json_schema\n schema_path: correction_set.v1.json\n")}
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{
"definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n",
})
_, err := newRuntimeWithPromptFS(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}}, promptFS)
assetPath := path.Join(prompts.AssetRoot, "modules.grammar.proposal.yaml")
promptFS[assetPath] = &fstest.MapFile{Data: []byte("collections:\n audita:\n prompts:\n - id: modules.grammar.proposal\n messages: []\n")}
loaded, err := config.Load(ck.Input{Source: ck.NoSource()}, "")
if err != nil {
t.Fatal(err)
}
_, err = newRuntimeWithPromptFS(context.Background(), loaded, nil, promptFS)
if err == nil || !IsAssetError(err) {
t.Fatalf("broken embedded prompt was not classified as an asset error: %v", err)
}
@@ -242,14 +69,24 @@ func TestBrokenEmbeddedPromptFailsEmbeddedOnlyConstruction(t *testing.T) {
func TestOperatorPromptCanRepairInvalidEmbeddedPrompt(t *testing.T) {
promptFS := copyEmbeddedPromptFilesystem(t)
assetPath := path.Join(prompts.AssetRoot, prompts.PromptIDModuleGrammarProposal+".yaml")
promptFS[assetPath] = &fstest.MapFile{Data: []byte("prompts:\n modules.grammar.proposal:\n v1:\n messages: []\n output:\n format: json\n validation_mode: json_schema\n schema_path: correction_set.v1.json\n")}
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{
"operator.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\nprompts:\n modules.grammar.proposal:\n v1:\n messages:\n - role: system\n content: operator system\n - role: user\n content: operator user\n",
})
runtime, err := newRuntimeWithPromptFS(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}}, promptFS)
assetPath := path.Join(prompts.AssetRoot, "modules.grammar.proposal.yaml")
promptFS[assetPath] = &fstest.MapFile{Data: []byte("collections:\n audita:\n prompts:\n - id: modules.grammar.proposal\n messages: []\n")}
dir := t.TempDir()
for name, content := range map[string]string{
"config.yml": "version: 3\n",
"prompts.yml": "collections:\n audita:\n prompts:\n - id: modules.grammar.proposal\n messages:\n - role: user\n content: operator message\n output:\n format: text\n validation_mode: none\n",
} {
if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0600); err != nil {
t.Fatal(err)
}
}
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
if err != nil {
t.Fatalf("valid final configuration was rejected with an invalid displaced embedded prompt: %v", err)
t.Fatal(err)
}
runtime, err := newRuntimeWithPromptFS(context.Background(), loaded, nil, promptFS)
if err != nil {
t.Fatalf("valid final configuration rejected: %v", err)
}
if runtime.Engine == nil {
t.Fatal("runtime engine was not retained")

258
internal/framework/llm/runtime_inspection_test.go
View File

@@ -9,258 +9,60 @@ import (
"strings"
"testing"
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gopkg.in/yaml.v3"
ck "go.maximumdirect.net/configkit"
)
type rejectingTransport struct{ t *testing.T }
func (r rejectingTransport) RoundTrip(*http.Request) (*http.Response, error) {
r.t.Error("profile inspection contacted a provider")
if r.t != nil {
r.t.Error("resource inspection contacted a provider")
}
return nil, errors.New("provider access forbidden")
}
func profileLayout(t *testing.T, base, overlay string) string {
t.Helper()
dir := t.TempDir()
for _, name := range []string{"profiles", "overlays"} {
if err := os.Mkdir(filepath.Join(dir, name), 0700); err != nil {
t.Fatal(err)
}
}
for name, content := range map[string]string{
"base": "profiles:\n base:\n backend: offline\n model: full-model\n temperature: 0.3\n max_tokens: 500\n top_p: 0.8\n timeout_seconds: 12\n service_tier: priority\n reasoning_effort: medium\n",
"testing": "profiles:\n testing:\n backend: offline\n model: economy-model\n",
} {
if err := os.WriteFile(filepath.Join(dir, "profiles", name+".yaml"), []byte(content), 0600); err != nil {
t.Fatal(err)
}
}
if err := os.WriteFile(filepath.Join(dir, "config.yml"), []byte(base), 0600); err != nil {
t.Fatal(err)
}
if overlay != "" {
if err := os.WriteFile(filepath.Join(dir, "overlays", "testing.yml"), []byte(overlay), 0600); err != nil {
t.Fatal(err)
}
}
return dir
}
const profileBase = "version: 2\nbackends:\n offline:\n endpoint: https://offline.invalid/v1/\n credential: SYNTHETIC_KEY\nllm:\n proposal_profile: base\n"
func TestOfflineProfileAssemblyAndSafeViews(t *testing.T) {
t.Setenv("SYNTHETIC_KEY", "synthetic-value-never-print")
dir := profileLayout(t, profileBase, "llm:\n proposal_profile: testing\n")
client := &http.Client{Transport: rejectingTransport{t}}
for _, tc := range []struct {
name, overlay string
overrides []ck.Override
want string
}{
{"base", "", nil, "full-model"},
{"overlay", "testing", nil, "economy-model"},
{"explicit override", "testing", []ck.Override{{Path: ck.Keys("llm", "proposal_profile"), Value: "base", Flag: "--proposal-profile"}}, "full-model"},
} {
t.Run(tc.name, func(t *testing.T) {
main := filepath.Join(dir, "config.yml")
loaded, err := config.Load(ck.File(main), tc.overlay, tc.overrides...)
if err != nil {
t.Fatal(err)
}
runtime, err := NewRuntime(context.Background(), loaded.Config, main, client)
if err != nil {
t.Fatal(err)
}
if runtime.Engine == nil || runtime.Profiles["proposal"].EffectiveModelParams.Model != tc.want || runtime.Profiles["validation"].EffectiveModelParams.Model != tc.want {
t.Fatalf("roles not resolved: %+v", runtime.Profiles)
}
profiles, err := runtime.SelectedProfilesYAML()
if err != nil {
t.Fatal(err)
}
explain, err := ExplanationYAML(loaded)
if err != nil {
t.Fatal(err)
}
combined := string(profiles) + string(explain)
if !strings.Contains(combined, tc.want) || !strings.Contains(combined, "/llm/validation_profile") {
t.Fatalf("inspection missing selection/provenance: %s", combined)
}
if strings.Contains(combined, "synthetic-value-never-print") || strings.Contains(combined, "https://offline.invalid") {
t.Fatal("safe views exposed credentials or endpoint")
}
if tc.name == "overlay" && !strings.Contains(combined, "overlay") {
t.Fatal("overlay origin missing")
}
if tc.name == "explicit override" && !strings.Contains(combined, "cli") {
t.Fatal("CLI origin missing")
}
if !strings.Contains(string(explain), "validation uses the final proposal profile when omitted") {
t.Fatal("inherited validation derivation missing")
}
if tc.name == "overlay" && !strings.Contains(string(explain), "source: overlays/testing.yml\n layer: overlay") {
t.Fatal("overlay field source missing")
}
if tc.name == "explicit override" && !strings.Contains(string(explain), "source: --proposal-profile\n layer: cli") {
t.Fatal("explicit profile source missing")
}
if tc.name == "base" && (!strings.Contains(string(profiles), "temperature: 0.3") || !strings.Contains(string(profiles), "reasoning_effort: medium")) {
t.Fatal("selected controls missing")
}
})
}
}
func TestSafeViewsOmitProviderAndPromptValues(t *testing.T) {
const providerValue = "private-provider-value"
const promptValue = "private-operator-prompt-text"
runtime, _, dir := adapterFixtureWithDocuments(t, nil, 0, 0, nil, "offline", map[string]string{
"definitions.yaml": "profiles:\n base:\n backend: offline\n model: full-model\n extra_params:\n vendor_extension: " + providerValue + "\n",
"operator-prompt.yaml": operatorPromptDocument(promptValue, promptValue),
})
profiles, err := runtime.SelectedProfilesYAML()
if err != nil {
t.Fatal(err)
}
loaded, err := config.Load(ck.File(filepath.Join(dir, "config.yml")), "")
if err != nil {
t.Fatal(err)
}
explain, err := ExplanationYAML(loaded)
if err != nil {
t.Fatal(err)
}
combined := string(profiles) + string(explain)
for _, value := range []string{providerValue, promptValue, "adapter-secret", "127.0.0.1"} {
if strings.Contains(combined, value) {
t.Fatalf("safe views exposed %q: %s", value, combined)
}
}
}
func TestExplanationYAMLIncludesSafeInheritanceEvidence(t *testing.T) {
t.Setenv("SYNTHETIC_KEY", "synthetic-inherited-secret-never-print")
text := `version: 2
backends:
shared:
endpoint: https://shared.invalid/v1
concurrency: 3
queue_capacity: 0
credential: SYNTHETIC_KEY
offline:
extends: shared
concurrency: 1
llm:
proposal_profile: base
data := `version: 3
glossary_collections:
common:
entries:
- id: a
name: secret-value-not-for-explanation
category: character
derived:
extends: common
`
loaded, err := config.Load(ck.Bytes("config.yml", []byte(text)), "")
if err != nil {
file := filepath.Join(t.TempDir(), "config.yml")
if err := os.WriteFile(file, []byte(data), 0600); err != nil {
t.Fatal(err)
}
loaded, err := config.Load(ck.Input{Source: ck.File(file)}, "")
if err != nil {
t.Fatalf("%v: %+v", err, err.(*ck.LoadError).Diagnostics())
}
explanation, err := ExplanationYAML(loaded)
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(explanation), "synthetic-inherited-secret-never-print") {
t.Fatal("explanation exposed a resolved inherited credential")
text := string(explanation)
if strings.Contains(text, "secret-value-not-for-explanation") {
t.Fatal("explanation exposed an effective value")
}
var records []struct {
Path string `yaml:"path"`
Inheritance []struct {
From string `yaml:"from"`
To string `yaml:"to"`
Reference string `yaml:"reference"`
} `yaml:"inheritance"`
}
if err := yaml.Unmarshal(explanation, &records); err != nil {
t.Fatal(err)
}
var endpoint, credential, concurrency *struct {
Path string `yaml:"path"`
Inheritance []struct {
From string `yaml:"from"`
To string `yaml:"to"`
Reference string `yaml:"reference"`
} `yaml:"inheritance"`
}
for i := range records {
switch records[i].Path {
case "/backends/offline/endpoint":
endpoint = &records[i]
case "/backends/offline/credential":
credential = &records[i]
case "/backends/offline/concurrency":
concurrency = &records[i]
}
}
if endpoint == nil || len(endpoint.Inheritance) != 1 {
t.Fatalf("inherited endpoint evidence missing: %s", explanation)
}
step := endpoint.Inheritance[0]
if step.From != "/backends/shared/endpoint" || step.To != "/backends/offline/endpoint" || step.Reference != "/backends/offline/extends" {
t.Fatalf("inherited endpoint evidence: %+v", step)
}
if credential == nil || len(credential.Inheritance) != 1 {
t.Fatalf("inherited credential evidence missing: %s", explanation)
}
if concurrency == nil || len(concurrency.Inheritance) != 0 {
t.Fatalf("direct child override reported parent inheritance: %+v", concurrency)
if !strings.Contains(text, "/glossary_collections/derived/entries") || !strings.Contains(text, "inheritance:") || !strings.Contains(text, "/glossary_collections/common/entries") {
t.Fatalf("inherited field evidence missing: %s", text)
}
}
func TestProfileAssemblyRejectsUnsafeSelections(t *testing.T) {
t.Setenv("SYNTHETIC_KEY", "synthetic-value-never-print")
dir := profileLayout(t, profileBase, "")
main := filepath.Join(dir, "config.yml")
for _, tc := range []struct{ name, profile string }{
{"endpoint only", "profiles:\n custom:\n endpoint: https://offline.invalid/v1\n model: model\n"},
{"mismatched endpoint", "profiles:\n custom:\n backend: offline\n endpoint: https://elsewhere.invalid/v1\n model: model\n"},
{"profile credential", "profiles:\n custom:\n backend: offline\n model: model\n api_key_env: OTHER_KEY\n"},
} {
t.Run(tc.name, func(t *testing.T) {
if err := os.WriteFile(filepath.Join(dir, "profiles", "custom.yaml"), []byte(tc.profile), 0600); err != nil {
t.Fatal(err)
}
loaded, err := config.Load(ck.File(main), "", ck.Override{Path: ck.Keys("llm", "proposal_profile"), Value: "custom", Flag: "--proposal-profile"})
if err != nil {
t.Fatal(err)
}
if _, err := NewRuntime(context.Background(), loaded.Config, main, &http.Client{Transport: rejectingTransport{t}}); err == nil {
t.Fatal("unsafe profile accepted")
}
})
}
loaded, err := config.Load(ck.File(main), "", ck.Override{Path: ck.Keys("llm", "proposal_profile"), Value: "missing", Flag: "--proposal-profile"})
func TestRuntimeInspectionDoesNotRequireCredentialsOrProvider(t *testing.T) {
loaded, err := config.Load(ck.Input{Source: ck.NoSource()}, "")
if err != nil {
t.Fatal(err)
}
if _, err := NewRuntime(context.Background(), loaded.Config, main, nil); err == nil {
t.Fatal("missing profile accepted")
if _, err := NewRuntime(context.Background(), loaded, &http.Client{Transport: rejectingTransport{t}}); err != nil {
t.Fatal(err)
}
if got := RedactError(errors.New("provider: synthetic-value-never-print"), loaded.Config); strings.Contains(got, "synthetic-value-never-print") {
t.Fatalf("secret leaked: %q", got)
}
}
func TestAbsoluteProfileDirectoryAndMissingSecret(t *testing.T) {
t.Setenv("SYNTHETIC_KEY", "synthetic-value-never-print")
dir := profileLayout(t, profileBase, "")
main := filepath.Join(dir, "config.yml")
absolute := strings.Replace(profileBase, "version: 2\n", "version: 2\nprofiles_dir: "+filepath.Join(dir, "profiles")+"\n", 1)
if err := os.WriteFile(main, []byte(absolute), 0600); err != nil {
t.Fatal(err)
}
loaded, err := config.Load(ck.File(main), "")
if err != nil {
t.Fatal(err)
}
if _, err := NewRuntime(context.Background(), loaded.Config, main, &http.Client{Transport: rejectingTransport{t}}); err != nil {
t.Fatal(err)
}
t.Setenv("SYNTHETIC_KEY", "")
if _, err := config.Load(ck.File(main), ""); err == nil {
t.Fatal("missing surviving secret accepted")
if _, err := ExplanationYAML(nil); err == nil {
t.Fatal("nil configuration accepted")
}
}

34
internal/framework/llm/secrets_test.go
View File

@@ -1,31 +1,45 @@
package llm
import (
"context"
"os"
"path/filepath"
"testing"
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
ck "go.maximumdirect.net/configkit"
)
func TestConfiguredSecretsIncludesUnselectedBackends(t *testing.T) {
if ConfiguredSecrets(nil) != nil {
t.Fatal("nil configuration has secrets")
var empty *Runtime
if empty.ConfiguredSecrets(context.Background()) != nil {
t.Fatal("nil runtime has secrets")
}
t.Setenv("SELECTED_KEY", "selected-secret")
t.Setenv("OTHER_KEY", "unselected-secret")
loaded, err := config.Load(ck.Bytes("config.yml", []byte("version: 2\nbackends:\n selected:\n endpoint: https://selected.invalid/v1\n credential: SELECTED_KEY\n other:\n endpoint: https://other.invalid/v1\n credential: OTHER_KEY\nllm:\n proposal_profile: base\n")), "")
t.Setenv("UNRELATED_KEY", "unrelated-secret")
dir := t.TempDir()
for name, data := range map[string]string{
"config.yml": "version: 3\n",
"prompts.yml": "backends:\n - id: selected\n endpoint: https://selected.invalid/v1\n api_key_env: SELECTED_KEY\n - id: other\n endpoint: https://other.invalid/v1\n api_key_env: OTHER_KEY\nprofiles:\n - id: base\n backend: selected\n model: example\n",
} {
if err := os.WriteFile(filepath.Join(dir, name), []byte(data), 0600); err != nil {
t.Fatal(err)
}
}
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
if err != nil {
t.Fatal(err)
}
got := ConfiguredSecrets(&loaded.Config)
if len(got) != 2 {
t.Fatalf("secrets count = %d", len(got))
runtime, err := NewRuntime(context.Background(), loaded, nil)
if err != nil {
t.Fatal(err)
}
seen := map[string]bool{}
for _, value := range got {
for _, value := range runtime.ConfiguredSecrets(context.Background()) {
seen[value] = true
}
if !seen["selected-secret"] || !seen["unselected-secret"] {
t.Fatal("missing resolved backend secret")
if len(seen) != 2 || !seen["selected-secret"] || !seen["unselected-secret"] {
t.Fatalf("unexpected resolved backend secrets: %v", seen)
}
}

56
internal/framework/llm/source_paths_test.go
View File

@@ -2,58 +2,42 @@ package llm
import (
"context"
"net/http"
"os"
"path/filepath"
"reflect"
"sort"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
ck "go.maximumdirect.net/configkit"
)
func TestRuntimeTracksCapturedConfigurationAndInheritedMessageFiles(t *testing.T) {
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n validation_profile: validation\n", map[string]string{
"selected.yml": "profiles:\n proposal:\n backend: offline\n model: proposal\n validation:\n backend: offline\n model: validation\n",
})
profiles := filepath.Join(filepath.Dir(main), cfg.ProfilesDir)
parentDir := filepath.Join(profiles, "catalog")
childDir := filepath.Join(profiles, "nested", "child")
for _, directory := range []string{parentDir, childDir} {
if err := os.MkdirAll(directory, 0700); err != nil {
t.Fatal(err)
}
}
parent := filepath.Join(parentDir, "arbitrary-name.yaml")
child := filepath.Join(childDir, "override.yaml")
resource := filepath.Join(parentDir, "instruction.md")
parentDocument := "prompts:\n source:\n release/1~draft:\n messages:\n - role: user\n content_file: ./instruction.md\n output:\n format: text\n validation_mode: basic\n"
childDocument := "prompts:\n literal/~child:\n extends: source\n"
for path, content := range map[string]string{
parent: parentDocument,
child: childDocument,
resource: "Captured operator message.\n",
dir := t.TempDir()
for name, content := range map[string]string{
"config.yml": "version: 3\n",
"prompts.yml": "config:\n imports:\n - path: catalog/arbitrary-name.yaml\n required: true\n",
"catalog/arbitrary-name.yaml": "prompts:\n - id: source\n messages:\n - role: user\n content_file: instruction.md\n output:\n format: text\n validation_mode: none\n",
"catalog/instruction.md": "Captured operator message.\n",
"prompts/nested/override.yaml": "prompts:\n - id: literal-child\n extends: {id: source}\n",
} {
if err := os.WriteFile(path, []byte(content), 0600); err != nil {
file := filepath.Join(dir, name)
if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(file, []byte(content), 0600); err != nil {
t.Fatal(err)
}
}
runtime, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{}})
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
if err != nil {
t.Fatal(err)
}
want := []string{
filepath.Join(profiles, "nested", "child", "override.yaml"),
filepath.Join(profiles, "catalog", "arbitrary-name.yaml"),
filepath.Join(profiles, "selected.yml"),
resource,
}
// Runtime paths are canonical paths from PromptKit's captured sources.
for i := range want {
want[i], err = filepath.EvalSymlinks(want[i])
if err != nil {
t.Fatal(err)
}
runtime, err := NewRuntime(context.Background(), loaded, nil)
if err != nil {
t.Fatal(err)
}
want := []string{filepath.Join(dir, "prompts.yml"), filepath.Join(dir, "catalog/arbitrary-name.yaml"), filepath.Join(dir, "catalog/instruction.md"), filepath.Join(dir, "prompts/nested/override.yaml")}
sort.Strings(want)
if !reflect.DeepEqual(runtime.SourcePaths, want) {
t.Fatalf("captured source paths = %#v, want %#v", runtime.SourcePaths, want)