eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Complete assembled validation and migrate remaining tests

Browse Source
This commit is contained in:
Eric Rakestraw
2026-11-10 18:13:12 +00:00
parent ba1d5d2367
commit 2844ce7e6a
44 changed files with 577 additions and 1624 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
internal/framework/contracts/chunk_proposal_composition_test.go
View File

@@ -23,12 +23,7 @@ func (h testChunkProposalHarness) collectEnrichedProposals(
transcript *schema.Transcript,
sections []chunking.Section,
) ([]proposals.EnrichedCorrectionProposal, error) {
runSpecs, err := ResolveModuleRunSpecs([]string{h.module.Key()})
if err != nil {
return nil, err
}
runSpec := runSpecs[0]
runSpec := ModuleRunSpec{ModuleKey: h.module.Key(), InstanceName: h.module.Key()}
runSpec.ReplacementPolicy = h.module.ReplacementPolicy()
out := make([]proposals.EnrichedCorrectionProposal, 0)
@@ -209,7 +204,7 @@ func TestApplyFakeProposalsAcrossMultipleChunksWithStaleSkip(t *testing.T) {
t.Fatalf("collectEnrichedProposals failed: %v", err)
}
result := proposals.ApplyProposals(transcript, enriched, module.ReplacementPolicy())
result := proposals.ApplyProposals(transcript, enriched, module.ReplacementPolicy(), nil)
if len(result.Applied) != 2 {
t.Fatalf("expected 2 applied changes, got %d", len(result.Applied))
@@ -249,7 +244,7 @@ func TestChunkProposalCompositionDoesNotMutateNormalizedTranscript(t *testing.T)
t.Fatalf("collectEnrichedProposals failed: %v", err)
}
_ = proposals.ApplyProposals(transcript, enriched, module.ReplacementPolicy())
_ = proposals.ApplyProposals(transcript, enriched, module.ReplacementPolicy(), nil)
if !reflect.DeepEqual(transcript, before) {
t.Fatalf("normalized transcript mutated\nbefore=%+v\nafter=%+v", before, transcript)

29
internal/framework/contracts/contracts_test.go
View File

@@ -7,6 +7,7 @@ import (
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
)
@@ -80,28 +81,14 @@ func TestInterfaceContractsCompileWithFakes(t *testing.T) {
}
func TestResolveModuleRunSpecs(t *testing.T) {
specs, err := ResolveModuleRunSpecs([]string{"glossary", "homophones", "glossary", "spoken_word", "grammar"})
if err != nil {
t.Fatalf("unexpected error: %v", err)
id := "named"
cfg := &config.Config{Pipeline: config.Pipeline{Stages: []config.Stage{{Module: "glossary", ID: &id}, {Module: "grammar"}, {Module: "glossary"}}}}
specs := ResolveModuleRunSpecs(cfg)
if len(specs) != 3 || specs[0].Index != 0 || specs[0].ID == nil || *specs[0].ID != id || specs[0].InstanceName != "glossary_1" || specs[1].InstanceName != "grammar_2" || specs[2].InstanceName != "glossary_3" {
t.Fatalf("unexpected specs: %+v", specs)
}
expected := []ModuleRunSpec{
{ModuleKey: "glossary", InstanceName: "glossary_1"},
{ModuleKey: "homophones", InstanceName: "homophones"},
{ModuleKey: "glossary", InstanceName: "glossary_2"},
{ModuleKey: "spoken_word", InstanceName: "spoken_word"},
{ModuleKey: "grammar", InstanceName: "grammar"},
}
if !reflect.DeepEqual(specs, expected) {
t.Fatalf("unexpected specs\nexpected: %+v\nactual: %+v", expected, specs)
}
}
func TestResolveModuleRunSpecsRejectsEmptyKey(t *testing.T) {
_, err := ResolveModuleRunSpecs([]string{"glossary", ""})
if err == nil {
t.Fatal("expected error for empty module key")
if len(ResolveModuleRunSpecs(nil)) != 0 {
t.Fatal("nil configuration has run specs")
}
}

3
internal/framework/llm/adapter.go
View File

@@ -100,6 +100,9 @@ func (c *RoleClient) CompleteStructured(parent context.Context, req structuredou
return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "unknown structured response schema", Cause: contractErr}
}
vars := prompts.WithSharedVariables(req.Variables)
for name, value := range prompts.SharedVariables() {
vars[name] = value
}
handle, prepErr := c.runtime.Engine.PrepareExecution(ctx, pk.RunRequest{
PromptID: req.PromptID, ProfileID: c.profileID, Vars: vars,
TransientRetries: &c.settings.TransientRetries,

215
internal/framework/llm/adapter_capacity_test.go
View File

@@ -2,216 +2,121 @@ package llm
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
pk "gitea.maximumdirect.net/eric/promptkit"
ck "go.maximumdirect.net/configkit"
)
func TestRolesShareBackendCapacityAndReleaseAfterCancellation(t *testing.T) {
t.Setenv("INHERITED_TEST_KEY", "synthetic-inherited-runtime-secret")
t.Setenv("OPERATOR_BACKEND_KEY", "synthetic-operator-backend-secret")
entered := make(chan struct{}, 3)
release := make(chan struct{})
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if got := r.Header.Get("Authorization"); got != "Bearer synthetic-inherited-runtime-secret" {
t.Errorf("inherited credential header: %q", got)
}
entered <- struct{}{}
<-release
content, _ := json.Marshal(`{"corrections":[]}`)
_, _ = fmt.Fprintf(w, `{"choices":[{"message":{"content":%s}}]}`, content)
}))
defer server.Close()
func capacityFixture(t *testing.T, resources string, transport http.RoundTripper) *Runtime {
t.Helper()
dir := t.TempDir()
if err := os.Mkdir(filepath.Join(dir, "profiles"), 0700); err != nil {
t.Fatal(err)
}
main := filepath.Join(dir, "config.yml")
body := fmt.Sprintf("version: 2\nbackends:\n shared-base:\n endpoint: %s/v1\n credential: INHERITED_TEST_KEY\n concurrency: 1\n queue_capacity: 0\n shared:\n extends: shared-base\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n repair_attempts: 0\n", server.URL)
if err := os.WriteFile(main, []byte(body), 0600); err != nil {
t.Fatal(err)
}
for _, name := range []string{"proposal", "validation"} {
profile := "profiles:\n " + name + ":\n backend: shared\n model: local\n"
if err := os.WriteFile(filepath.Join(dir, "profiles", name+".yaml"), []byte(profile), 0600); err != nil {
for name, content := range map[string]string{"config.yml": "version: 3\n", "prompts.yml": resources} {
if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0600); err != nil {
t.Fatal(err)
}
}
operatorBackend := `backends:
shared:
endpoint: https://operator.invalid/v1
concurrency_limit: 77
queue_capacity: 88
api_key_env: OPERATOR_BACKEND_KEY
`
if err := os.WriteFile(filepath.Join(dir, "profiles", "backend-tuning.yaml"), []byte(operatorBackend), 0600); err != nil {
t.Fatal(err)
}
loaded, err := config.Load(ck.File(main), "")
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
if err != nil {
t.Fatal(err)
}
runtime, err := NewRuntime(context.Background(), loaded.Config, main, server.Client())
runtime, err := NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
if err != nil {
t.Fatal(err)
}
proposal, err := runtime.NewRoleClient("proposal", CallHooks{})
return runtime
}
func capacityResponse(r *http.Request) *http.Response {
return &http.Response{StatusCode: 200, Header: make(http.Header), Request: r, Body: io.NopCloser(strings.NewReader(`{"choices":[{"message":{"content":"{\"corrections\":[]}"}}]}`))}
}
func TestRolesShareBackendCapacityAndReleaseAfterCancellation(t *testing.T) {
entered := make(chan struct{}, 3)
release := make(chan struct{})
transport := transportFunc(func(r *http.Request) (*http.Response, error) {
entered <- struct{}{}
select {
case <-release:
return capacityResponse(r), nil
case <-r.Context().Done():
return nil, r.Context().Err()
}
})
resources := "backends:\n - id: shared\n endpoint: https://example.invalid/v1\n concurrency_limit: 1\n queue_capacity: 1\nprofiles:\n - id: proposal\n backend: shared\n model: local\n - id: validation\n backend: shared\n model: local\n"
runtime := capacityFixture(t, resources, transport)
settings := config.LLM{OperationTimeout: 3 * time.Second}
proposal, err := runtime.NewRoleClient("proposal", "proposal", settings, CallHooks{})
if err != nil {
t.Fatal(err)
}
validation, err := runtime.NewRoleClient("validation", CallHooks{})
validation, err := runtime.NewRoleClient("validation", "validation", settings, CallHooks{})
if err != nil {
t.Fatal(err)
}
request := structuredoutput.Request{PromptID: prompts.PromptIDModuleGlossaryProposal, PromptVersion: prompts.VersionV1, Variables: map[string]string{"TranscriptDescriptionBlock": "", "GlossaryJSON": "[]", "SectionJSON": "[]"}, Schema: responseschema.CorrectionSetKey}
firstDone := make(chan error, 1)
request := adapterRequest(responseschema.CorrectionSetKey)
done := make(chan error, 1)
go func() {
var output map[string]any
_, callErr := proposal.CompleteStructured(context.Background(), request, &output)
firstDone <- callErr
_, err := proposal.CompleteStructured(context.Background(), request, &map[string]any{})
done <- err
}()
select {
case <-entered:
case <-time.After(2 * time.Second):
t.Fatal("first call did not enter provider")
}
var output map[string]any
capacityResponse, err := validation.CompleteStructured(context.Background(), request, &output)
if !errors.Is(err, pk.ErrCapacityExceeded) {
t.Fatalf("other role did not share capacity: %v", err)
}
var capacityFailure *structuredoutput.Failure
if !errors.As(err, &capacityFailure) || capacityFailure.Class != structuredoutput.FailureRecoverable || capacityFailure.Reason != outcomes.ReasonCapacityExceeded || !capacityResponse.Execution.Available || capacityResponse.Execution.ProviderAttempts != 0 {
t.Fatalf("capacity outcome: failure=%#v counts=%+v", capacityFailure, capacityResponse.Execution)
}
select {
case <-entered:
t.Fatal("second role reached provider despite full backend")
default:
}
close(release)
if err := <-firstDone; err != nil {
t.Fatal(err)
}
_, err = validation.CompleteStructured(context.Background(), request, &output)
if err != nil {
t.Fatalf("capacity was not released: %v", err)
}
for len(entered) > 0 {
<-entered
}
release = make(chan struct{})
if err := os.WriteFile(main, []byte(strings.Replace(body, "queue_capacity: 0", "queue_capacity: 1", 1)), 0600); err != nil {
t.Fatal(err)
}
loaded, err = config.Load(ck.File(main), "")
if err != nil {
t.Fatal(err)
}
runtime, err = NewRuntime(context.Background(), loaded.Config, main, server.Client())
if err != nil {
t.Fatal(err)
}
proposal, err = runtime.NewRoleClient("proposal", CallHooks{})
if err != nil {
t.Fatal(err)
}
validation, err = runtime.NewRoleClient("validation", CallHooks{})
if err != nil {
t.Fatal(err)
}
go func() {
var out map[string]any
_, callErr := proposal.CompleteStructured(context.Background(), request, &out)
firstDone <- callErr
}()
select {
case <-entered:
case <-time.After(2 * time.Second):
t.Fatal("first queued-capacity call did not dispatch")
}
queuedCtx, cancelQueued := context.WithTimeout(context.Background(), 30*time.Millisecond)
defer cancelQueued()
_, err = validation.CompleteStructured(queuedCtx, request, &output)
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond)
defer cancel()
_, err = validation.CompleteStructured(ctx, request, &map[string]any{})
if !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("queued call did not honor deadline: %v", err)
}
if !errors.As(err, &capacityFailure) || capacityFailure.Class != structuredoutput.FailureCanceled || capacityFailure.Reason != outcomes.ReasonCanceled {
t.Fatalf("queued parent deadline classification: %#v", capacityFailure)
select {
case <-entered:
t.Fatal("queued call dispatched")
default:
}
close(release)
if err := <-firstDone; err != nil {
if err := <-done; err != nil {
t.Fatal(err)
}
_, err = validation.CompleteStructured(context.Background(), request, &output)
if err != nil {
t.Fatalf("queued cancellation leaked capacity: %v", err)
if _, err := validation.CompleteStructured(context.Background(), request, &map[string]any{}); err != nil {
t.Fatalf("cancellation leaked capacity: %v", err)
}
}
func TestDistinctBackendsHaveIndependentCapacity(t *testing.T) {
entered := make(chan struct{}, 2)
release := make(chan struct{})
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
transport := transportFunc(func(r *http.Request) (*http.Response, error) {
entered <- struct{}{}
<-release
schema := map[string]any{}
_ = json.NewDecoder(r.Body).Decode(&schema)
content, _ := json.Marshal(`{"corrections":[]}`)
_, _ = fmt.Fprintf(w, `{"choices":[{"message":{"content":%s}}]}`, content)
}))
defer server.Close()
dir := t.TempDir()
if err := os.Mkdir(filepath.Join(dir, "profiles"), 0700); err != nil {
t.Fatal(err)
}
main := filepath.Join(dir, "config.yml")
body := fmt.Sprintf("version: 2\nbackends:\n first:\n endpoint: %s/v1\n concurrency: 1\n queue_capacity: 0\n second:\n endpoint: %s/v1\n concurrency: 1\n queue_capacity: 0\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n repair_attempts: 0\n", server.URL, server.URL)
if err := os.WriteFile(main, []byte(body), 0600); err != nil {
t.Fatal(err)
}
for name, backend := range map[string]string{"proposal": "first", "validation": "second"} {
if err := os.WriteFile(filepath.Join(dir, "profiles", name+".yaml"), []byte("profiles:\n "+name+":\n backend: "+backend+"\n model: local\n"), 0600); err != nil {
t.Fatal(err)
}
}
loaded, err := config.Load(ck.File(main), "")
return capacityResponse(r), nil
})
resources := "backends:\n - id: first\n endpoint: https://example.invalid/v1\n concurrency_limit: 1\n queue_capacity: 0\n - id: second\n endpoint: https://example.invalid/v1\n concurrency_limit: 1\n queue_capacity: 0\nprofiles:\n - id: proposal\n backend: first\n model: local\n - id: validation\n backend: second\n model: local\n"
runtime := capacityFixture(t, resources, transport)
settings := config.LLM{OperationTimeout: 3 * time.Second}
first, err := runtime.NewRoleClient("proposal", "proposal", settings, CallHooks{})
if err != nil {
t.Fatal(err)
}
runtime, err := NewRuntime(context.Background(), loaded.Config, main, server.Client())
second, err := runtime.NewRoleClient("validation", "validation", settings, CallHooks{})
if err != nil {
t.Fatal(err)
}
proposal, err := runtime.NewRoleClient("proposal", CallHooks{})
if err != nil {
t.Fatal(err)
}
validation, err := runtime.NewRoleClient("validation", CallHooks{})
if err != nil {
t.Fatal(err)
}
request := structuredoutput.Request{PromptID: prompts.PromptIDModuleGlossaryProposal, PromptVersion: prompts.VersionV1, Variables: map[string]string{"TranscriptDescriptionBlock": "", "GlossaryJSON": "[]", "SectionJSON": "[]"}, Schema: responseschema.CorrectionSetKey}
request := adapterRequest(responseschema.CorrectionSetKey)
done := make(chan error, 2)
go func() {
var out map[string]any
_, e := proposal.CompleteStructured(context.Background(), request, &out)
done <- e
_, err := first.CompleteStructured(context.Background(), request, &map[string]any{})
done <- err
}()
select {
case <-entered:
@@ -219,13 +124,13 @@ func TestDistinctBackendsHaveIndependentCapacity(t *testing.T) {
t.Fatal("first backend did not dispatch")
}
go func() {
var out map[string]any
_, e := validation.CompleteStructured(context.Background(), request, &out)
done <- e
_, err := second.CompleteStructured(context.Background(), request, &map[string]any{})
done <- err
}()
select {
case <-entered:
case <-time.After(2 * time.Second):
close(release)
t.Fatal("second backend was blocked by first")
}
close(release)

267
internal/framework/llm/adapter_test.go
View File

@@ -5,8 +5,8 @@ import (
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
@@ -15,13 +15,13 @@ import (
"testing"
"time"
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
pk "gitea.maximumdirect.net/eric/promptkit"
ck "go.maximumdirect.net/configkit"
pk "go.maximumdirect.net/promptkit"
)
type reply struct {
@@ -30,85 +30,56 @@ type reply struct {
}
func adapterFixture(t *testing.T, replies []reply, retries, repairs int, hook func(*http.Request)) (*Runtime, *atomic.Int32, string) {
return adapterFixtureWithDocuments(t, replies, retries, repairs, hook, "offline", map[string]string{
"definitions.yaml": "profiles:\n base:\n backend: offline\n model: full-model\n",
})
return adapterFixtureWithPrompt(t, replies, retries, repairs, hook, "")
}
func adapterFixtureWithPrompt(t *testing.T, replies []reply, retries, repairs int, hook func(*http.Request), operatorPrompt string) (*Runtime, *atomic.Int32, string) {
documents := map[string]string{
"definitions.yaml": "profiles:\n base:\n backend: offline\n model: full-model\n",
}
if operatorPrompt != "" {
documents["operator-prompt.yaml"] = operatorPrompt
}
return adapterFixtureWithDocuments(t, replies, retries, repairs, hook, "offline", documents)
}
func adapterFixtureWithDocuments(t *testing.T, replies []reply, retries, repairs int, hook func(*http.Request), backendID string, documents map[string]string) (*Runtime, *atomic.Int32, string) {
t.Helper()
var count atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
t.Setenv("ADAPTER_KEY", "adapter-secret")
dir := t.TempDir()
resources := "backends:\n - id: offline\n endpoint: https://provider.invalid/v1\n api_key_env: ADAPTER_KEY\n concurrency_limit: 1\n queue_capacity: 0\nprofiles:\n - id: base\n backend: offline\n model: full-model\n"
for name, body := range map[string]string{"config.yml": "version: 3\n", "prompts.yml": resources + operatorPrompt} {
if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0600); err != nil {
t.Fatal(err)
}
}
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
if err != nil {
t.Fatal(err)
}
transport := transportFunc(func(r *http.Request) (*http.Response, error) {
call := int(count.Add(1))
if hook != nil {
hook(r)
}
step := replies[len(replies)-1]
if call <= len(replies) {
step = replies[call-1]
if err := r.Context().Err(); err != nil {
return nil, err
}
step := replies[min(call-1, len(replies)-1)]
if step.status == 0 {
connection, _, hijackErr := w.(http.Hijacker).Hijack()
if hijackErr != nil {
t.Error(hijackErr)
return
return nil, errors.New("synthetic transport error")
}
body := `{"error":{"message":"provider error"}}`
if step.status == http.StatusOK {
body = `{"choices":[]}`
if step.content != "MALFORMED_ENVELOPE" {
encoded, _ := json.Marshal(step.content)
body = fmt.Sprintf(`{"model":"returned-model","choices":[{"message":{"content":%s}}],"usage":{"prompt_tokens":2,"completion_tokens":3,"total_tokens":5}}`, encoded)
}
_ = connection.Close()
return
}
if step.status != http.StatusOK {
w.WriteHeader(step.status)
_, _ = fmt.Fprint(w, `{"error":{"message":"provider error"}}`)
return
}
if step.content == "MALFORMED_ENVELOPE" {
_, _ = fmt.Fprint(w, `{"choices":[]}`)
return
}
encoded, _ := json.Marshal(step.content)
w.Header().Set("Content-Type", "application/json")
_, _ = fmt.Fprintf(w, `{"model":"returned-model","choices":[{"message":{"content":%s}}],"usage":{"prompt_tokens":2,"completion_tokens":3,"total_tokens":5}}`, encoded)
}))
t.Cleanup(server.Close)
t.Setenv("ADAPTER_KEY", "adapter-secret")
dir := t.TempDir()
profileDirectory := filepath.Join(dir, "profiles")
if err := os.Mkdir(profileDirectory, 0700); err != nil {
t.Fatal(err)
}
base := fmt.Sprintf("version: 2\nprofiles_dir: profiles\nbackends:\n %s:\n endpoint: %s/v1\n credential: ADAPTER_KEY\n concurrency: 1\n queue_capacity: 0\nllm:\n proposal_profile: base\n transient_retries: %d\n repair_attempts: %d\n", backendID, server.URL, retries, repairs)
main := filepath.Join(dir, "config.yml")
if err := os.WriteFile(main, []byte(base), 0600); err != nil {
t.Fatal(err)
}
for name, content := range documents {
if err := os.WriteFile(filepath.Join(profileDirectory, name), []byte(content), 0600); err != nil {
t.Fatal(err)
}
}
loaded, err := config.Load(ck.File(main), "")
if err != nil {
t.Fatal(err)
}
runtime, err := NewRuntime(context.Background(), loaded.Config, main, server.Client())
return &http.Response{StatusCode: step.status, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
})
runtime, err := NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
if err != nil {
t.Fatal(err)
}
runtime.Config.LLM = config.LLM{OperationTimeout: 10 * time.Second, TransientRetries: retries, RepairAttempts: repairs}
return runtime, &count, dir
}
func operatorPromptDocument(system, user string) string {
return fmt.Sprintf("prompts:\n %s:\n v1:\n messages:\n - role: system\n content: %q\n - role: user\n content: %q\n output:\n format: text\n validation_mode: basic\n", prompts.PromptIDModuleGlossaryProposal, system, user)
return fmt.Sprintf("collections:\n audita:\n prompts:\n - id: modules.glossary.proposal\n messages:\n - role: system\n content: %q\n - role: user\n content: %q\n output:\n format: text\n validation_mode: none\n", system, user)
}
func adapterRequest(schema responseschema.Key) structuredoutput.Request {
@@ -118,7 +89,7 @@ func adapterRequest(schema responseschema.Key) structuredoutput.Request {
id = prompts.PromptIDValidatorEditorialReview
vars = map[string]string{"TranscriptDescriptionBlock": "", "PayloadJSON": "[]"}
}
return structuredoutput.Request{PromptID: id, PromptVersion: prompts.VersionV1, Variables: vars, Schema: schema, Context: structuredoutput.DiagnosticContext{ModuleKey: "glossary"}}
return structuredoutput.Request{PromptID: id, Variables: vars, Schema: schema, Context: structuredoutput.DiagnosticContext{ModuleKey: "glossary"}}
}
func TestPreparedAdapterRepairsAndClassifies(t *testing.T) {
@@ -154,7 +125,7 @@ func TestPreparedAdapterRepairsAndClassifies(t *testing.T) {
}
var events []pk.ExecutionEvent
var records []structuredoutput.CallRecord
client, err := runtime.NewRoleClient(role, CallHooks{OnEvent: func(e pk.ExecutionEvent) { events = append(events, e) }, OnCall: func(r structuredoutput.CallRecord) { records = append(records, r) }})
client, err := runtime.NewRoleClient(role, "base", runtime.Config.LLM, CallHooks{OnEvent: func(e pk.ExecutionEvent) { events = append(events, e) }, OnCall: func(r structuredoutput.CallRecord) { records = append(records, r) }})
if err != nil {
t.Fatal(err)
}
@@ -234,7 +205,7 @@ func TestCorrectionSchemaReplacementValuesThroughAdapter(t *testing.T) {
}
payload := `{"corrections":[{` + strings.Join(fields, ",") + `}]}`
runtime, count, _ := adapterFixture(t, []reply{{200, payload}}, 0, tc.repairs, nil)
client, err := runtime.NewRoleClient("proposal", CallHooks{})
client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{})
if err != nil {
t.Fatal(err)
}
@@ -280,7 +251,7 @@ func TestPreparedAdapterMetadataAndTargetGuard(t *testing.T) {
t.Error(err)
}
})
client, err := runtime.NewRoleClient("proposal", CallHooks{})
client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{})
if err != nil {
t.Fatal(err)
}
@@ -314,12 +285,7 @@ func TestPreparedAdapterMetadataAndTargetGuard(t *testing.T) {
if !found {
t.Fatal("hardening policy missing")
}
backend := runtime.Config.Backends["offline"]
backend.Endpoint = "http://127.0.0.1:1/v1"
runtime.Config.Backends["offline"] = backend
if _, err := client.CompleteStructured(context.Background(), req, &out); err == nil || !strings.Contains(err.Error(), "target changed") || count.Load() != 1 {
t.Fatalf("changed application backend target dispatched: %v, %d", err, count.Load())
}
}
func TestOperatorPromptOverridesUsePerCallSchemaAndOptionalSharedText(t *testing.T) {
@@ -365,7 +331,7 @@ func TestOperatorPromptOverridesUsePerCallSchemaAndOptionalSharedText(t *testing
}
posted = append(posted, envelope)
}, operatorPromptDocument(tc.system, tc.user))
client, err := runtime.NewRoleClient("proposal", CallHooks{})
client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{})
if err != nil {
t.Fatal(err)
}
@@ -403,150 +369,9 @@ func TestOperatorPromptOverridesUsePerCallSchemaAndOptionalSharedText(t *testing
}
}
func TestProfilePresenceBackendAuthorityAndProviderParametersReachWire(t *testing.T) {
t.Setenv("CATALOG_TRAP_KEY", "catalog-secret-must-not-be-used")
cases := []struct {
name string
parentTemperature string
childTemperature string
wantTemperature bool
}{
{name: "explicit zero", parentTemperature: " temperature: 0.8\n", childTemperature: " temperature: 0\n", wantTemperature: true},
{name: "omitted", wantTemperature: false},
{name: "null clears inherited control", parentTemperature: " temperature: 0.8\n", childTemperature: " temperature: null\n", wantTemperature: false},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
profileDocument := "profiles:\n \"parent~/partial\":\n backend: openrouter\n model: parent-model\n" + tc.parentTemperature +
" base:\n extends: \"parent~/partial\"\n model: full-model\n timeout_seconds: 0\n" + tc.childTemperature +
" extra_params:\n routing:\n overridden: profile\n profile_only: retained\n cleared: null\n"
operatorBackend := `backends:
openrouter:
endpoint: https://operator.invalid/v1
concurrency_limit: 91
queue_capacity: 92
api_key_env: CATALOG_TRAP_KEY
extra_params:
routing:
inherited: backend
overridden: backend
cleared: remove-me
`
documents := map[string]string{
"unrelated-profile-file.yml": profileDocument,
"provider-tuning.yaml": operatorBackend,
}
var wire map[string]json.RawMessage
runtime, count, _ := adapterFixtureWithDocuments(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, func(r *http.Request) {
if got := r.Header.Get("Authorization"); got != "Bearer adapter-secret" {
t.Errorf("application credential was not authoritative: %q", got)
}
if err := json.NewDecoder(r.Body).Decode(&wire); err != nil {
t.Error(err)
}
}, "openrouter", documents)
profile := runtime.Profiles["proposal"]
if profile.EffectiveModelParams.APIKeyEnv != "" || profile.EffectiveModelParams.Endpoint == "https://operator.invalid/v1" || profile.EffectiveModelParams.TimeoutSeconds != 0 {
t.Fatalf("application backend mapping or zero timeout lost: %+v", profile.EffectiveModelParams)
}
client, err := runtime.NewRoleClient("proposal", CallHooks{})
if err != nil {
t.Fatal(err)
}
var out map[string]any
response, err := client.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &out)
if err != nil {
t.Fatal(err)
}
if count.Load() != 1 {
t.Fatalf("provider calls = %d, want 1", count.Load())
}
temperature, hasTemperature := wire["temperature"]
if hasTemperature != tc.wantTemperature {
t.Fatalf("temperature presence = %t, want %t: %s", hasTemperature, tc.wantTemperature, wire)
}
if tc.wantTemperature {
var value float64
if err := json.Unmarshal(temperature, &value); err != nil || value != 0 {
t.Fatalf("explicit zero temperature = %s (%v)", temperature, err)
}
}
var routing map[string]any
if err := json.Unmarshal(wire["routing"], &routing); err != nil {
t.Fatalf("provider routing fields missing: %s (%v)", wire, err)
}
if routing["inherited"] != "backend" || routing["overridden"] != "profile" || routing["profile_only"] != "retained" {
t.Fatalf("recursively overlaid provider fields = %#v", routing)
}
if value, exists := routing["cleared"]; !exists || value != nil {
t.Fatalf("null did not replace inherited provider field: %#v", routing)
}
if response.Model != "full-model" || response.Prepared == nil {
t.Fatalf("execution did not use selected profile: %+v", response)
}
})
}
}
func TestPreparedAdapterUsesCapturedProfileAndPromptUntilReconstructed(t *testing.T) {
var posted []struct {
Model string `json:"model"`
Messages []structuredoutput.Message `json:"messages"`
}
runtime, count, dir := adapterFixtureWithPrompt(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, func(r *http.Request) {
var envelope struct {
Model string `json:"model"`
Messages []structuredoutput.Message `json:"messages"`
}
if err := json.NewDecoder(r.Body).Decode(&envelope); err != nil {
t.Error(err)
return
}
posted = append(posted, envelope)
}, operatorPromptDocument("Initial operator instructions.", "Initial payload."))
client, err := runtime.NewRoleClient("proposal", CallHooks{})
if err != nil {
t.Fatal(err)
}
profile := "profiles:\n base:\n backend: offline\n model: edited-model\n"
if err := os.WriteFile(filepath.Join(dir, "profiles", "definitions.yaml"), []byte(profile), 0600); err != nil {
t.Fatal(err)
}
updatedPrompt := operatorPromptDocument("Edited operator instructions.", "Edited payload.")
if err := os.WriteFile(filepath.Join(dir, "profiles", "operator-prompt.yaml"), []byte(updatedPrompt), 0600); err != nil {
t.Fatal(err)
}
var out map[string]any
response, err := client.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &out)
if err != nil {
t.Fatal(err)
}
if response.Model != "full-model" || response.Prepared == nil || response.Prepared.Messages[0].Content != "Initial operator instructions." || posted[0].Model != "full-model" {
t.Fatalf("existing runtime did not retain its captured configuration: response=%+v request=%+v", response, posted[0])
}
main := filepath.Join(dir, "config.yml")
newRuntime, err := NewRuntime(context.Background(), runtime.Config, main, http.DefaultClient)
if err != nil {
t.Fatal(err)
}
newClient, err := newRuntime.NewRoleClient("proposal", CallHooks{})
if err != nil {
t.Fatal(err)
}
response, err = newClient.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &out)
if err != nil {
t.Fatal(err)
}
if int(count.Load()) != 2 || response.Model != "edited-model" || response.Prepared == nil || response.Prepared.Messages[0].Content != "Edited operator instructions." || posted[1].Model != "edited-model" {
t.Fatalf("new runtime did not capture edited configuration: response=%+v request=%+v", response, posted[1])
}
}
func TestPreparedAdapterRejectsTargetDriftAndPreparationErrors(t *testing.T) {
runtime, count, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, nil)
client, err := runtime.NewRoleClient("proposal", CallHooks{})
client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{})
if err != nil {
t.Fatal(err)
}
@@ -563,7 +388,7 @@ func TestPreparedAdapterCancellationStopsRetryWait(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
var once sync.Once
client, err := runtime.NewRoleClient("proposal", CallHooks{OnEvent: func(e pk.ExecutionEvent) {
client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{OnEvent: func(e pk.ExecutionEvent) {
if e.Kind == pk.ExecutionAttempt && e.Dispatched {
once.Do(cancel)
}
@@ -580,11 +405,9 @@ func TestPreparedAdapterCancellationStopsRetryWait(t *testing.T) {
func TestPreparedAdapterDistinguishesOperationTimeoutFromParentCancellation(t *testing.T) {
blocking := func(*http.Request) { time.Sleep(500 * time.Millisecond) }
runtime, _, _ := adapterFixtureWithDocuments(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, blocking, "offline", map[string]string{
"definitions.yaml": "profiles:\n base:\n backend: offline\n model: full-model\n timeout_seconds: 0\n",
})
runtime, _, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, blocking)
runtime.Config.LLM.OperationTimeout = 200 * time.Millisecond
client, err := runtime.NewRoleClient("proposal", CallHooks{})
client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{})
if err != nil {
t.Fatal(err)
}
@@ -605,7 +428,7 @@ func TestPreparedAdapterDistinguishesOperationTimeoutFromParentCancellation(t *t
func TestPreparedAdapterClassifiesSetupAndDomainDecodeFailures(t *testing.T) {
runtime, _, _ := adapterFixture(t, []reply{{200, `{"validations":[{"correction_index":9999999999999999999999999,"approved":true,"confidence":0.8,"reason":"ok"}]}`}}, 0, 0, nil)
client, err := runtime.NewRoleClient("validation", CallHooks{})
client, err := runtime.NewRoleClient("validation", "base", runtime.Config.LLM, CallHooks{})
if err != nil {
t.Fatal(err)
}
@@ -638,7 +461,7 @@ func TestPreparedAdapterClassifiesSetupAndDomainDecodeFailures(t *testing.T) {
func TestPreparedAdapterNonRetryableProviderStatus(t *testing.T) {
runtime, count, _ := adapterFixture(t, []reply{{http.StatusBadRequest, ""}}, 2, 0, nil)
client, err := runtime.NewRoleClient("proposal", CallHooks{})
client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{})
if err != nil {
t.Fatal(err)
}

4
internal/framework/llm/observations_test.go
View File

@@ -10,7 +10,7 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
pk "gitea.maximumdirect.net/eric/promptkit"
pk "go.maximumdirect.net/promptkit"
)
func TestObservationSummarySeparatesLogicalCallsAttemptsAndUsage(t *testing.T) {
@@ -93,7 +93,7 @@ func TestObservationCollectorConcurrentAttributionAndSnapshotCopy(t *testing.T)
func TestRoleClientFeedsCollectorAcrossPreparationFailure(t *testing.T) {
runtime, count, _ := adapterFixture(t, []reply{{status: 200, content: `{"corrections":[]}`}}, 0, 0, nil)
collector := NewObservationCollector()
client, err := runtime.NewRoleClient("proposal", collector.Hooks())
client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, collector.Hooks())
if err != nil {
t.Fatal(err)
}

225
internal/framework/llm/runtime_assembly_test.go
View File

@@ -4,43 +4,17 @@ import (
"context"
"errors"
"io/fs"
"net/http"
"os"
"path"
"path/filepath"
"strings"
"testing"
"testing/fstest"
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
ck "go.maximumdirect.net/configkit"
)
func runtimeAssemblyFixture(t *testing.T, selections string, documents map[string]string) (config.Config, string) {
t.Helper()
dir := t.TempDir()
profileDir := filepath.Join(dir, "profiles")
if err := os.Mkdir(profileDir, 0700); err != nil {
t.Fatal(err)
}
main := filepath.Join(dir, "config.yml")
text := "version: 2\nprofiles_dir: profiles\nbackends:\n offline:\n endpoint: http://127.0.0.1:1/v1\n concurrency: 1\n queue_capacity: 0\nllm:\n" + selections
if err := os.WriteFile(main, []byte(text), 0600); err != nil {
t.Fatal(err)
}
for name, content := range documents {
if err := os.WriteFile(filepath.Join(profileDir, name), []byte(content), 0600); err != nil {
t.Fatal(err)
}
}
loaded, err := config.Load(ck.File(main), "")
if err != nil {
t.Fatal(err)
}
return loaded.Config, main
}
func copyEmbeddedPromptFilesystem(t *testing.T) fstest.MapFS {
t.Helper()
source := prompts.Filesystem()
@@ -62,166 +36,18 @@ func copyEmbeddedPromptFilesystem(t *testing.T) fstest.MapFS {
return files
}
func TestNewRuntimeRequiresCompleteSelectedProfiles(t *testing.T) {
validProposal := " backend: offline\n model: proposal-model\n"
cases := []struct {
name string
selections string
profiles string
role string
field string
}{
{
name: "omitted proposal model with derived validation selection",
selections: " proposal_profile: proposal\n",
profiles: "profiles:\n proposal:\n backend: offline\n",
role: "proposal",
field: "model",
},
{
name: "empty validation model",
selections: " proposal_profile: proposal\n validation_profile: validation\n",
profiles: "profiles:\n proposal:\n" + validProposal + " validation:\n backend: offline\n model: \"\"\n",
role: "validation",
field: "model",
},
{
name: "null proposal model",
selections: " proposal_profile: proposal\n",
profiles: "profiles:\n proposal:\n backend: offline\n model: null\n",
role: "proposal",
field: "model",
},
{
name: "cleared validation endpoint",
selections: " proposal_profile: proposal\n validation_profile: validation\n",
profiles: "profiles:\n proposal:\n" + validProposal + " validation:\n backend: offline\n endpoint: null\n model: validation-model\n",
role: "validation",
field: "endpoint",
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
cfg, main := runtimeAssemblyFixture(t, tc.selections, map[string]string{"definitions.yml": tc.profiles})
_, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}})
if err == nil || !strings.Contains(err.Error(), tc.role) || !strings.Contains(err.Error(), tc.field) {
t.Fatalf("selected incomplete profile was not rejected for %s.%s: %v", tc.role, tc.field, err)
}
})
}
}
func TestNewRuntimeAllowsUnselectedPartialProfiles(t *testing.T) {
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{
"definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n unfinished:\n backend: offline\n",
})
runtime, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}})
if err != nil {
t.Fatal(err)
}
if runtime.Profiles["proposal"].EffectiveModelParams.Model != "proposal-model" || runtime.Profiles["validation"].EffectiveModelParams.Model != "proposal-model" {
t.Fatalf("role profiles were not resolved: %+v", runtime.Profiles)
}
}
func TestNativeProfileInheritancePreservesControlsAndMergesProviderValues(t *testing.T) {
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: \"child profile\"\n", map[string]string{
"unrelated-name.yaml": `profiles:
"parent~/shared":
backend: offline
temperature: 0.4
top_p: 0.7
extra_params:
routing:
retained: from-parent
overridden: parent
cleared: remove-me
"child profile":
extends: "parent~/shared"
model: child-model
extra_params:
routing:
overridden: child
child_only: present
cleared: null
`,
})
runtime, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{}})
if err != nil {
t.Fatal(err)
}
profile := runtime.Profiles["proposal"]
target := profile.EffectiveModelParams
if profile.ProfileID != "child profile" || target.Model != "child-model" || target.Temperature != 0.4 || target.TopP != 0.7 {
t.Fatalf("inherited profile target = %+v", target)
}
routing, ok := target.ExtraParams["routing"].(map[string]any)
if !ok || routing["retained"] != "from-parent" || routing["overridden"] != "child" || routing["child_only"] != "present" {
t.Fatalf("recursively inherited provider values = %#v", target.ExtraParams)
}
if value, exists := routing["cleared"]; !exists || value != nil {
t.Fatalf("null did not replace the inherited provider value: %#v", routing)
}
}
func TestMissingSelectedProfileDirectoryIsOperatorInputError(t *testing.T) {
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{
"definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n",
})
if err := os.RemoveAll(filepath.Join(filepath.Dir(main), cfg.ProfilesDir)); err != nil {
t.Fatal(err)
}
_, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}})
if err == nil || IsAssetError(err) {
t.Fatalf("missing operator profile directory classification: %v", err)
}
}
func TestNewRuntimeRejectsMalformedUnselectedProfilesAndConflicts(t *testing.T) {
cases := []struct {
name string
document map[string]string
selection string
}{
{
name: "malformed unselected definition",
document: map[string]string{"definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n unused:\n temperature: 9\n"},
selection: " proposal_profile: proposal\n",
},
{
name: "undefined selected backend",
document: map[string]string{"definitions.yml": "profiles:\n proposal:\n backend: missing\n model: proposal-model\n"},
selection: " proposal_profile: proposal\n",
},
{
name: "conflicting directory contributions",
document: map[string]string{
"a.yml": "profiles:\n proposal:\n backend: offline\n model: first-model\n",
"b.yml": "profiles:\n proposal:\n model: second-model\n",
},
selection: " proposal_profile: proposal\n",
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
cfg, main := runtimeAssemblyFixture(t, tc.selection, tc.document)
_, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}})
if err == nil || IsAssetError(err) {
t.Fatalf("operator definition classification: %v", err)
}
})
}
}
func TestRuntimeClassifiesEmbeddedAssetsAndHonorsCancellation(t *testing.T) {
_, err := newRuntimeWithPromptFS(context.Background(), config.Config{}, "config.yml", nil, fstest.MapFS{})
loaded, err := config.Load(ck.Input{Source: ck.NoSource()}, "")
if err != nil {
t.Fatal(err)
}
_, err = newRuntimeWithPromptFS(context.Background(), loaded, nil, fstest.MapFS{})
if err == nil || !IsAssetError(err) || !errors.Is(err, fs.ErrNotExist) {
t.Fatalf("embedded discovery error was not preserved as an asset error: %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
_, err = newRuntimeWithPromptFS(ctx, config.Config{}, "config.yml", nil, fstest.MapFS{})
_, err = newRuntimeWithPromptFS(ctx, loaded, nil, fstest.MapFS{})
if !errors.Is(err, context.Canceled) || IsAssetError(err) {
t.Fatalf("caller cancellation classification: %v", err)
}
@@ -229,12 +55,13 @@ func TestRuntimeClassifiesEmbeddedAssetsAndHonorsCancellation(t *testing.T) {
func TestBrokenEmbeddedPromptFailsEmbeddedOnlyConstruction(t *testing.T) {
promptFS := copyEmbeddedPromptFilesystem(t)
assetPath := path.Join(prompts.AssetRoot, prompts.PromptIDModuleGrammarProposal+".yaml")
promptFS[assetPath] = &fstest.MapFile{Data: []byte("prompts:\n modules.grammar.proposal:\n v1:\n messages: []\n output:\n format: json\n validation_mode: json_schema\n schema_path: correction_set.v1.json\n")}
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{
"definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n",
})
_, err := newRuntimeWithPromptFS(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}}, promptFS)
assetPath := path.Join(prompts.AssetRoot, "modules.grammar.proposal.yaml")
promptFS[assetPath] = &fstest.MapFile{Data: []byte("collections:\n audita:\n prompts:\n - id: modules.grammar.proposal\n messages: []\n")}
loaded, err := config.Load(ck.Input{Source: ck.NoSource()}, "")
if err != nil {
t.Fatal(err)
}
_, err = newRuntimeWithPromptFS(context.Background(), loaded, nil, promptFS)
if err == nil || !IsAssetError(err) {
t.Fatalf("broken embedded prompt was not classified as an asset error: %v", err)
}
@@ -242,14 +69,24 @@ func TestBrokenEmbeddedPromptFailsEmbeddedOnlyConstruction(t *testing.T) {
func TestOperatorPromptCanRepairInvalidEmbeddedPrompt(t *testing.T) {
promptFS := copyEmbeddedPromptFilesystem(t)
assetPath := path.Join(prompts.AssetRoot, prompts.PromptIDModuleGrammarProposal+".yaml")
promptFS[assetPath] = &fstest.MapFile{Data: []byte("prompts:\n modules.grammar.proposal:\n v1:\n messages: []\n output:\n format: json\n validation_mode: json_schema\n schema_path: correction_set.v1.json\n")}
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{
"operator.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\nprompts:\n modules.grammar.proposal:\n v1:\n messages:\n - role: system\n content: operator system\n - role: user\n content: operator user\n",
})
runtime, err := newRuntimeWithPromptFS(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}}, promptFS)
assetPath := path.Join(prompts.AssetRoot, "modules.grammar.proposal.yaml")
promptFS[assetPath] = &fstest.MapFile{Data: []byte("collections:\n audita:\n prompts:\n - id: modules.grammar.proposal\n messages: []\n")}
dir := t.TempDir()
for name, content := range map[string]string{
"config.yml": "version: 3\n",
"prompts.yml": "collections:\n audita:\n prompts:\n - id: modules.grammar.proposal\n messages:\n - role: user\n content: operator message\n output:\n format: text\n validation_mode: none\n",
} {
if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0600); err != nil {
t.Fatal(err)
}
}
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
if err != nil {
t.Fatalf("valid final configuration was rejected with an invalid displaced embedded prompt: %v", err)
t.Fatal(err)
}
runtime, err := newRuntimeWithPromptFS(context.Background(), loaded, nil, promptFS)
if err != nil {
t.Fatalf("valid final configuration rejected: %v", err)
}
if runtime.Engine == nil {
t.Fatal("runtime engine was not retained")

258
internal/framework/llm/runtime_inspection_test.go
View File

@@ -9,258 +9,60 @@ import (
"strings"
"testing"
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gopkg.in/yaml.v3"
ck "go.maximumdirect.net/configkit"
)
type rejectingTransport struct{ t *testing.T }
func (r rejectingTransport) RoundTrip(*http.Request) (*http.Response, error) {
r.t.Error("profile inspection contacted a provider")
if r.t != nil {
r.t.Error("resource inspection contacted a provider")
}
return nil, errors.New("provider access forbidden")
}
func profileLayout(t *testing.T, base, overlay string) string {
t.Helper()
dir := t.TempDir()
for _, name := range []string{"profiles", "overlays"} {
if err := os.Mkdir(filepath.Join(dir, name), 0700); err != nil {
t.Fatal(err)
}
}
for name, content := range map[string]string{
"base": "profiles:\n base:\n backend: offline\n model: full-model\n temperature: 0.3\n max_tokens: 500\n top_p: 0.8\n timeout_seconds: 12\n service_tier: priority\n reasoning_effort: medium\n",
"testing": "profiles:\n testing:\n backend: offline\n model: economy-model\n",
} {
if err := os.WriteFile(filepath.Join(dir, "profiles", name+".yaml"), []byte(content), 0600); err != nil {
t.Fatal(err)
}
}
if err := os.WriteFile(filepath.Join(dir, "config.yml"), []byte(base), 0600); err != nil {
t.Fatal(err)
}
if overlay != "" {
if err := os.WriteFile(filepath.Join(dir, "overlays", "testing.yml"), []byte(overlay), 0600); err != nil {
t.Fatal(err)
}
}
return dir
}
const profileBase = "version: 2\nbackends:\n offline:\n endpoint: https://offline.invalid/v1/\n credential: SYNTHETIC_KEY\nllm:\n proposal_profile: base\n"
func TestOfflineProfileAssemblyAndSafeViews(t *testing.T) {
t.Setenv("SYNTHETIC_KEY", "synthetic-value-never-print")
dir := profileLayout(t, profileBase, "llm:\n proposal_profile: testing\n")
client := &http.Client{Transport: rejectingTransport{t}}
for _, tc := range []struct {
name, overlay string
overrides []ck.Override
want string
}{
{"base", "", nil, "full-model"},
{"overlay", "testing", nil, "economy-model"},
{"explicit override", "testing", []ck.Override{{Path: ck.Keys("llm", "proposal_profile"), Value: "base", Flag: "--proposal-profile"}}, "full-model"},
} {
t.Run(tc.name, func(t *testing.T) {
main := filepath.Join(dir, "config.yml")
loaded, err := config.Load(ck.File(main), tc.overlay, tc.overrides...)
if err != nil {
t.Fatal(err)
}
runtime, err := NewRuntime(context.Background(), loaded.Config, main, client)
if err != nil {
t.Fatal(err)
}
if runtime.Engine == nil || runtime.Profiles["proposal"].EffectiveModelParams.Model != tc.want || runtime.Profiles["validation"].EffectiveModelParams.Model != tc.want {
t.Fatalf("roles not resolved: %+v", runtime.Profiles)
}
profiles, err := runtime.SelectedProfilesYAML()
if err != nil {
t.Fatal(err)
}
explain, err := ExplanationYAML(loaded)
if err != nil {
t.Fatal(err)
}
combined := string(profiles) + string(explain)
if !strings.Contains(combined, tc.want) || !strings.Contains(combined, "/llm/validation_profile") {
t.Fatalf("inspection missing selection/provenance: %s", combined)
}
if strings.Contains(combined, "synthetic-value-never-print") || strings.Contains(combined, "https://offline.invalid") {
t.Fatal("safe views exposed credentials or endpoint")
}
if tc.name == "overlay" && !strings.Contains(combined, "overlay") {
t.Fatal("overlay origin missing")
}
if tc.name == "explicit override" && !strings.Contains(combined, "cli") {
t.Fatal("CLI origin missing")
}
if !strings.Contains(string(explain), "validation uses the final proposal profile when omitted") {
t.Fatal("inherited validation derivation missing")
}
if tc.name == "overlay" && !strings.Contains(string(explain), "source: overlays/testing.yml\n layer: overlay") {
t.Fatal("overlay field source missing")
}
if tc.name == "explicit override" && !strings.Contains(string(explain), "source: --proposal-profile\n layer: cli") {
t.Fatal("explicit profile source missing")
}
if tc.name == "base" && (!strings.Contains(string(profiles), "temperature: 0.3") || !strings.Contains(string(profiles), "reasoning_effort: medium")) {
t.Fatal("selected controls missing")
}
})
}
}
func TestSafeViewsOmitProviderAndPromptValues(t *testing.T) {
const providerValue = "private-provider-value"
const promptValue = "private-operator-prompt-text"
runtime, _, dir := adapterFixtureWithDocuments(t, nil, 0, 0, nil, "offline", map[string]string{
"definitions.yaml": "profiles:\n base:\n backend: offline\n model: full-model\n extra_params:\n vendor_extension: " + providerValue + "\n",
"operator-prompt.yaml": operatorPromptDocument(promptValue, promptValue),
})
profiles, err := runtime.SelectedProfilesYAML()
if err != nil {
t.Fatal(err)
}
loaded, err := config.Load(ck.File(filepath.Join(dir, "config.yml")), "")
if err != nil {
t.Fatal(err)
}
explain, err := ExplanationYAML(loaded)
if err != nil {
t.Fatal(err)
}
combined := string(profiles) + string(explain)
for _, value := range []string{providerValue, promptValue, "adapter-secret", "127.0.0.1"} {
if strings.Contains(combined, value) {
t.Fatalf("safe views exposed %q: %s", value, combined)
}
}
}
func TestExplanationYAMLIncludesSafeInheritanceEvidence(t *testing.T) {
t.Setenv("SYNTHETIC_KEY", "synthetic-inherited-secret-never-print")
text := `version: 2
backends:
shared:
endpoint: https://shared.invalid/v1
concurrency: 3
queue_capacity: 0
credential: SYNTHETIC_KEY
offline:
extends: shared
concurrency: 1
llm:
proposal_profile: base
data := `version: 3
glossary_collections:
common:
entries:
- id: a
name: secret-value-not-for-explanation
category: character
derived:
extends: common
`
loaded, err := config.Load(ck.Bytes("config.yml", []byte(text)), "")
if err != nil {
file := filepath.Join(t.TempDir(), "config.yml")
if err := os.WriteFile(file, []byte(data), 0600); err != nil {
t.Fatal(err)
}
loaded, err := config.Load(ck.Input{Source: ck.File(file)}, "")
if err != nil {
t.Fatalf("%v: %+v", err, err.(*ck.LoadError).Diagnostics())
}
explanation, err := ExplanationYAML(loaded)
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(explanation), "synthetic-inherited-secret-never-print") {
t.Fatal("explanation exposed a resolved inherited credential")
text := string(explanation)
if strings.Contains(text, "secret-value-not-for-explanation") {
t.Fatal("explanation exposed an effective value")
}
var records []struct {
Path string `yaml:"path"`
Inheritance []struct {
From string `yaml:"from"`
To string `yaml:"to"`
Reference string `yaml:"reference"`
} `yaml:"inheritance"`
}
if err := yaml.Unmarshal(explanation, &records); err != nil {
t.Fatal(err)
}
var endpoint, credential, concurrency *struct {
Path string `yaml:"path"`
Inheritance []struct {
From string `yaml:"from"`
To string `yaml:"to"`
Reference string `yaml:"reference"`
} `yaml:"inheritance"`
}
for i := range records {
switch records[i].Path {
case "/backends/offline/endpoint":
endpoint = &records[i]
case "/backends/offline/credential":
credential = &records[i]
case "/backends/offline/concurrency":
concurrency = &records[i]
}
}
if endpoint == nil || len(endpoint.Inheritance) != 1 {
t.Fatalf("inherited endpoint evidence missing: %s", explanation)
}
step := endpoint.Inheritance[0]
if step.From != "/backends/shared/endpoint" || step.To != "/backends/offline/endpoint" || step.Reference != "/backends/offline/extends" {
t.Fatalf("inherited endpoint evidence: %+v", step)
}
if credential == nil || len(credential.Inheritance) != 1 {
t.Fatalf("inherited credential evidence missing: %s", explanation)
}
if concurrency == nil || len(concurrency.Inheritance) != 0 {
t.Fatalf("direct child override reported parent inheritance: %+v", concurrency)
if !strings.Contains(text, "/glossary_collections/derived/entries") || !strings.Contains(text, "inheritance:") || !strings.Contains(text, "/glossary_collections/common/entries") {
t.Fatalf("inherited field evidence missing: %s", text)
}
}
func TestProfileAssemblyRejectsUnsafeSelections(t *testing.T) {
t.Setenv("SYNTHETIC_KEY", "synthetic-value-never-print")
dir := profileLayout(t, profileBase, "")
main := filepath.Join(dir, "config.yml")
for _, tc := range []struct{ name, profile string }{
{"endpoint only", "profiles:\n custom:\n endpoint: https://offline.invalid/v1\n model: model\n"},
{"mismatched endpoint", "profiles:\n custom:\n backend: offline\n endpoint: https://elsewhere.invalid/v1\n model: model\n"},
{"profile credential", "profiles:\n custom:\n backend: offline\n model: model\n api_key_env: OTHER_KEY\n"},
} {
t.Run(tc.name, func(t *testing.T) {
if err := os.WriteFile(filepath.Join(dir, "profiles", "custom.yaml"), []byte(tc.profile), 0600); err != nil {
t.Fatal(err)
}
loaded, err := config.Load(ck.File(main), "", ck.Override{Path: ck.Keys("llm", "proposal_profile"), Value: "custom", Flag: "--proposal-profile"})
if err != nil {
t.Fatal(err)
}
if _, err := NewRuntime(context.Background(), loaded.Config, main, &http.Client{Transport: rejectingTransport{t}}); err == nil {
t.Fatal("unsafe profile accepted")
}
})
}
loaded, err := config.Load(ck.File(main), "", ck.Override{Path: ck.Keys("llm", "proposal_profile"), Value: "missing", Flag: "--proposal-profile"})
func TestRuntimeInspectionDoesNotRequireCredentialsOrProvider(t *testing.T) {
loaded, err := config.Load(ck.Input{Source: ck.NoSource()}, "")
if err != nil {
t.Fatal(err)
}
if _, err := NewRuntime(context.Background(), loaded.Config, main, nil); err == nil {
t.Fatal("missing profile accepted")
if _, err := NewRuntime(context.Background(), loaded, &http.Client{Transport: rejectingTransport{t}}); err != nil {
t.Fatal(err)
}
if got := RedactError(errors.New("provider: synthetic-value-never-print"), loaded.Config); strings.Contains(got, "synthetic-value-never-print") {
t.Fatalf("secret leaked: %q", got)
}
}
func TestAbsoluteProfileDirectoryAndMissingSecret(t *testing.T) {
t.Setenv("SYNTHETIC_KEY", "synthetic-value-never-print")
dir := profileLayout(t, profileBase, "")
main := filepath.Join(dir, "config.yml")
absolute := strings.Replace(profileBase, "version: 2\n", "version: 2\nprofiles_dir: "+filepath.Join(dir, "profiles")+"\n", 1)
if err := os.WriteFile(main, []byte(absolute), 0600); err != nil {
t.Fatal(err)
}
loaded, err := config.Load(ck.File(main), "")
if err != nil {
t.Fatal(err)
}
if _, err := NewRuntime(context.Background(), loaded.Config, main, &http.Client{Transport: rejectingTransport{t}}); err != nil {
t.Fatal(err)
}
t.Setenv("SYNTHETIC_KEY", "")
if _, err := config.Load(ck.File(main), ""); err == nil {
t.Fatal("missing surviving secret accepted")
if _, err := ExplanationYAML(nil); err == nil {
t.Fatal("nil configuration accepted")
}
}

34
internal/framework/llm/secrets_test.go
View File

@@ -1,31 +1,45 @@
package llm
import (
"context"
"os"
"path/filepath"
"testing"
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
ck "go.maximumdirect.net/configkit"
)
func TestConfiguredSecretsIncludesUnselectedBackends(t *testing.T) {
if ConfiguredSecrets(nil) != nil {
t.Fatal("nil configuration has secrets")
var empty *Runtime
if empty.ConfiguredSecrets(context.Background()) != nil {
t.Fatal("nil runtime has secrets")
}
t.Setenv("SELECTED_KEY", "selected-secret")
t.Setenv("OTHER_KEY", "unselected-secret")
loaded, err := config.Load(ck.Bytes("config.yml", []byte("version: 2\nbackends:\n selected:\n endpoint: https://selected.invalid/v1\n credential: SELECTED_KEY\n other:\n endpoint: https://other.invalid/v1\n credential: OTHER_KEY\nllm:\n proposal_profile: base\n")), "")
t.Setenv("UNRELATED_KEY", "unrelated-secret")
dir := t.TempDir()
for name, data := range map[string]string{
"config.yml": "version: 3\n",
"prompts.yml": "backends:\n - id: selected\n endpoint: https://selected.invalid/v1\n api_key_env: SELECTED_KEY\n - id: other\n endpoint: https://other.invalid/v1\n api_key_env: OTHER_KEY\nprofiles:\n - id: base\n backend: selected\n model: example\n",
} {
if err := os.WriteFile(filepath.Join(dir, name), []byte(data), 0600); err != nil {
t.Fatal(err)
}
}
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
if err != nil {
t.Fatal(err)
}
got := ConfiguredSecrets(&loaded.Config)
if len(got) != 2 {
t.Fatalf("secrets count = %d", len(got))
runtime, err := NewRuntime(context.Background(), loaded, nil)
if err != nil {
t.Fatal(err)
}
seen := map[string]bool{}
for _, value := range got {
for _, value := range runtime.ConfiguredSecrets(context.Background()) {
seen[value] = true
}
if !seen["selected-secret"] || !seen["unselected-secret"] {
t.Fatal("missing resolved backend secret")
if len(seen) != 2 || !seen["selected-secret"] || !seen["unselected-secret"] {
t.Fatalf("unexpected resolved backend secrets: %v", seen)
}
}

56
internal/framework/llm/source_paths_test.go
View File

@@ -2,58 +2,42 @@ package llm
import (
"context"
"net/http"
"os"
"path/filepath"
"reflect"
"sort"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
ck "go.maximumdirect.net/configkit"
)
func TestRuntimeTracksCapturedConfigurationAndInheritedMessageFiles(t *testing.T) {
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n validation_profile: validation\n", map[string]string{
"selected.yml": "profiles:\n proposal:\n backend: offline\n model: proposal\n validation:\n backend: offline\n model: validation\n",
})
profiles := filepath.Join(filepath.Dir(main), cfg.ProfilesDir)
parentDir := filepath.Join(profiles, "catalog")
childDir := filepath.Join(profiles, "nested", "child")
for _, directory := range []string{parentDir, childDir} {
if err := os.MkdirAll(directory, 0700); err != nil {
t.Fatal(err)
}
}
parent := filepath.Join(parentDir, "arbitrary-name.yaml")
child := filepath.Join(childDir, "override.yaml")
resource := filepath.Join(parentDir, "instruction.md")
parentDocument := "prompts:\n source:\n release/1~draft:\n messages:\n - role: user\n content_file: ./instruction.md\n output:\n format: text\n validation_mode: basic\n"
childDocument := "prompts:\n literal/~child:\n extends: source\n"
for path, content := range map[string]string{
parent: parentDocument,
child: childDocument,
resource: "Captured operator message.\n",
dir := t.TempDir()
for name, content := range map[string]string{
"config.yml": "version: 3\n",
"prompts.yml": "config:\n imports:\n - path: catalog/arbitrary-name.yaml\n required: true\n",
"catalog/arbitrary-name.yaml": "prompts:\n - id: source\n messages:\n - role: user\n content_file: instruction.md\n output:\n format: text\n validation_mode: none\n",
"catalog/instruction.md": "Captured operator message.\n",
"prompts/nested/override.yaml": "prompts:\n - id: literal-child\n extends: {id: source}\n",
} {
if err := os.WriteFile(path, []byte(content), 0600); err != nil {
file := filepath.Join(dir, name)
if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(file, []byte(content), 0600); err != nil {
t.Fatal(err)
}
}
runtime, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{}})
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
if err != nil {
t.Fatal(err)
}
want := []string{
filepath.Join(profiles, "nested", "child", "override.yaml"),
filepath.Join(profiles, "catalog", "arbitrary-name.yaml"),
filepath.Join(profiles, "selected.yml"),
resource,
}
// Runtime paths are canonical paths from PromptKit's captured sources.
for i := range want {
want[i], err = filepath.EvalSymlinks(want[i])
if err != nil {
t.Fatal(err)
}
runtime, err := NewRuntime(context.Background(), loaded, nil)
if err != nil {
t.Fatal(err)
}
want := []string{filepath.Join(dir, "prompts.yml"), filepath.Join(dir, "catalog/arbitrary-name.yaml"), filepath.Join(dir, "catalog/instruction.md"), filepath.Join(dir, "prompts/nested/override.yaml")}
sort.Strings(want)
if !reflect.DeepEqual(runtime.SourcePaths, want) {
t.Fatalf("captured source paths = %#v, want %#v", runtime.SourcePaths, want)

24
internal/framework/modules/registry_test.go
View File

@@ -6,6 +6,7 @@ import (
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
@@ -41,17 +42,14 @@ func TestUnknownModuleKeyNotRecognized(t *testing.T) {
}
func TestRepeatedRunSpecNamingRemainsDeterministic(t *testing.T) {
specs, err := contracts.ResolveModuleRunSpecs([]string{"glossary", "glossary", "grammar"})
if err != nil {
t.Fatalf("ResolveModuleRunSpecs error: %v", err)
}
if specs[0].InstanceName != "glossary_1" || specs[1].InstanceName != "glossary_2" || specs[2].InstanceName != "grammar" {
specs := contracts.ResolveModuleRunSpecs(&config.Config{Pipeline: config.Pipeline{Stages: []config.Stage{{Module: "glossary"}, {Module: "glossary"}, {Module: "grammar"}}}})
if specs[0].InstanceName != "glossary_1" || specs[1].InstanceName != "glossary_2" || specs[2].InstanceName != "grammar_3" {
t.Fatalf("unexpected instance names: %+v", specs)
}
}
func TestUnsupportedUnknownModuleKeyFailsCleanly(t *testing.T) {
factory := NewFactory(Dependencies{})
factory := NewFactory()
_, err := factory.ModuleForSpec(contracts.ModuleRunSpec{ModuleKey: "unknown", InstanceName: "unknown"})
if err == nil {
t.Fatal("expected unsupported-module error")
@@ -67,7 +65,7 @@ func TestUnsupportedUnknownModuleKeyFailsCleanly(t *testing.T) {
}
func TestRecognizedButMissingConstructorFailsWithInternalRegistryError(t *testing.T) {
factory := NewFactory(Dependencies{})
factory := NewFactory()
// Force a missing constructor state for a known key.
factory.constructors[ModuleKeySpokenWord] = nil
@@ -81,7 +79,7 @@ func TestRecognizedButMissingConstructorFailsWithInternalRegistryError(t *testin
}
func TestGrammarIsRegisteredAndConstructibleByDefault(t *testing.T) {
factory := NewFactory(Dependencies{})
factory := NewFactory()
module, err := factory.ModuleForSpec(contracts.ModuleRunSpec{ModuleKey: ModuleKeyGrammar, InstanceName: ModuleKeyGrammar})
if err != nil {
t.Fatalf("ModuleForSpec error: %v", err)
@@ -92,7 +90,7 @@ func TestGrammarIsRegisteredAndConstructibleByDefault(t *testing.T) {
}
func TestGlossaryIsRegisteredAndConstructibleByDefault(t *testing.T) {
factory := NewFactory(Dependencies{})
factory := NewFactory()
module, err := factory.ModuleForSpec(contracts.ModuleRunSpec{ModuleKey: ModuleKeyGlossary, InstanceName: ModuleKeyGlossary})
if err != nil {
t.Fatalf("ModuleForSpec error: %v", err)
@@ -103,7 +101,7 @@ func TestGlossaryIsRegisteredAndConstructibleByDefault(t *testing.T) {
}
func TestHomophonesIsRegisteredAndConstructibleByDefault(t *testing.T) {
factory := NewFactory(Dependencies{})
factory := NewFactory()
module, err := factory.ModuleForSpec(contracts.ModuleRunSpec{ModuleKey: ModuleKeyHomophones, InstanceName: ModuleKeyHomophones})
if err != nil {
t.Fatalf("ModuleForSpec error: %v", err)
@@ -114,7 +112,7 @@ func TestHomophonesIsRegisteredAndConstructibleByDefault(t *testing.T) {
}
func TestSpokenWordIsRegisteredAndConstructibleByDefault(t *testing.T) {
factory := NewFactory(Dependencies{})
factory := NewFactory()
module, err := factory.ModuleForSpec(contracts.ModuleRunSpec{ModuleKey: ModuleKeySpokenWord, InstanceName: ModuleKeySpokenWord})
if err != nil {
t.Fatalf("ModuleForSpec error: %v", err)
@@ -125,7 +123,7 @@ func TestSpokenWordIsRegisteredAndConstructibleByDefault(t *testing.T) {
}
func TestRegisterConstructorAndConstruct(t *testing.T) {
factory := NewFactory(Dependencies{})
factory := NewFactory()
if err := factory.RegisterConstructor(ModuleKeyGlossary, func(ctx context.Context, req ConstructRequest) (contracts.TranscriptModule, error) {
_ = ctx
if req.RunSpec.InstanceName != "glossary_1" {
@@ -149,7 +147,7 @@ func TestRegisterConstructorAndConstruct(t *testing.T) {
}
func TestRegisterConstructorRejectsUnknownModuleKey(t *testing.T) {
factory := NewFactory(Dependencies{})
factory := NewFactory()
err := factory.RegisterConstructor("unknown", func(ctx context.Context, req ConstructRequest) (contracts.TranscriptModule, error) {
_ = ctx
_ = req

11
internal/framework/processreport/llm_test.go
View File

@@ -7,11 +7,10 @@ import (
"testing"
"time"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
pk "gitea.maximumdirect.net/eric/promptkit"
pk "go.maximumdirect.net/promptkit"
)
func TestExecutionReportingGroupsPredispatchFailureAndKeepsSafeMilliseconds(t *testing.T) {
@@ -24,7 +23,7 @@ func TestExecutionReportingGroupsPredispatchFailureAndKeepsSafeMilliseconds(t *t
Events: []llm.ExecutionObservation{{CallID: 1, Kind: string(pk.ExecutionFinished), BackendID: "primary", Attempts: 2, TransientRetries: 1, CapacityWait: 20 * time.Millisecond, GenerationDuration: 500 * time.Millisecond, RetryDelay: 100 * time.Millisecond}},
}
output := &runner.RunOutput{Observations: snapshot, ObservationsAvailable: true, LLMSummary: llm.Summarize(snapshot)}
utilization := BuildUtilization(output, map[string]config.Backend{"primary": {Concurrency: 2, QueueCapacity: 3}, "secondary": {Concurrency: 1}})
utilization := BuildUtilization(output, map[string]*pk.BackendInspection{"primary": {ConcurrencyLimit: intPointer(2), QueueCapacity: intPointer(3)}, "secondary": {ConcurrencyLimit: intPointer(1)}})
if utilization.Backends["primary"].LogicalCalls != 1 || utilization.Backends["primary"].ProviderAttempts != 2 || utilization.Backends["primary"].CapacityWaitMS != 20 || utilization.Backends["primary"].LogicalWallMS != 1500 {
t.Fatalf("primary backend: %+v", utilization.Backends["primary"])
}
@@ -66,7 +65,7 @@ func BenchmarkExecutionReports(b *testing.B) {
})
b.Run(fmt.Sprintf("utilization/%d", n), func(b *testing.B) {
for i := 0; i < b.N; i++ {
BuildUtilization(output, map[string]config.Backend{"local": {Concurrency: 1}})
BuildUtilization(output, map[string]*pk.BackendInspection{"local": {ConcurrencyLimit: intPointer(1)}})
}
})
}
@@ -110,7 +109,7 @@ func TestIndexedReportsPreserveOrderAndPartialUsage(t *testing.T) {
t.Fatalf("call usage: %+v", execution.Calls)
}
output := &runner.RunOutput{Observations: snapshot}
utilization := BuildUtilization(output, map[string]config.Backend{"a": {}, "b": {}})
utilization := BuildUtilization(output, map[string]*pk.BackendInspection{"a": {}, "b": {}})
total := utilization.Backends["a"].Usage.Total
if total.Value != 7 || !total.Known || total.Complete || utilization.Backends["a"].ProviderAttempts != 2 {
t.Fatalf("partial backend usage: %+v", utilization)
@@ -119,3 +118,5 @@ func TestIndexedReportsPreserveOrderAndPartialUsage(t *testing.T) {
t.Fatalf("module grouping: %+v", utilization.Modules)
}
}
func intPointer(value int) *int { return &value }

2
internal/framework/processreport/partial_validation_test.go
View File

@@ -70,7 +70,7 @@ func TestFatalBatchRetainsEarlierDecisionsInLedger(t *testing.T) {
cfg := config.Config{Chunking: config.Chunking{MaxSectionTokens: 8192, MinSectionTokens: 1}, Validation: config.Validation{MaxPromptTokens: 150}}
result, err := runner.New(partialModule{validator}).Run(context.Background(), runner.RunInput{
Config: &cfg, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}}, ValidationLLMClient: client, CaptureCandidates: true,
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}}, CaptureCandidates: true,
})
if !errors.Is(err, origin) || client.calls != 3 {
t.Fatalf("fatal cause/calls: %v %d", err, client.calls)

41
internal/framework/proposal_generation/adapter_integration_test.go
View File

@@ -4,73 +4,74 @@ import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
ck "go.maximumdirect.net/configkit"
)
type proposalTransport func(*http.Request) (*http.Response, error)
func (f proposalTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
func TestRepairedProposalStillFacesDomainChecks(t *testing.T) {
calls := 0
outputs := []string{`{"wrong":true}`, `{"corrections":[{"id":1,"original_text":"absent","corrected_text":"present","confidence":0.4}]}`}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
transport := proposalTransport(func(r *http.Request) (*http.Response, error) {
output := outputs[len(outputs)-1]
if calls < len(outputs) {
output = outputs[calls]
}
calls++
content, _ := json.Marshal(output)
_, _ = fmt.Fprintf(w, `{"choices":[{"message":{"content":%s}}]}`, content)
}))
defer server.Close()
body := fmt.Sprintf(`{"choices":[{"message":{"content":%s}}]}`, content)
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
})
dir := t.TempDir()
if err := os.Mkdir(filepath.Join(dir, "profiles"), 0700); err != nil {
t.Fatal(err)
}
main := filepath.Join(dir, "config.yml")
content := fmt.Sprintf("version: 2\nbackends:\n offline:\n endpoint: %s/v1\nllm:\n proposal_profile: base\n transient_retries: 0\n repair_attempts: 1\n", server.URL)
if err := os.WriteFile(main, []byte(content), 0600); err != nil {
if err := os.WriteFile(main, []byte("version: 3\n"), 0600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "profiles", "base.yaml"), []byte("profiles:\n base:\n backend: offline\n model: local-model\n"), 0600); err != nil {
if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte("profiles:\n - {id: base, endpoint: 'https://example.invalid/v1', model: local-model, api_key_required: false}\n"), 0600); err != nil {
t.Fatal(err)
}
loaded, err := config.Load(ck.File(main), "")
loaded, err := config.Load(ck.Input{Source: ck.File(main)}, "")
if err != nil {
t.Fatal(err)
}
runtime, err := llm.NewRuntime(context.Background(), loaded.Config, main, server.Client())
runtime, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
if err != nil {
t.Fatal(err)
}
client, err := runtime.NewRoleClient("proposal", llm.CallHooks{})
settings := loaded.Config.LLM
settings.TransientRetries, settings.RepairAttempts = 0, 1
client, err := runtime.NewRoleClient("proposal", "base", settings, llm.CallHooks{})
if err != nil {
t.Fatal(err)
}
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "gestures"}}}
vars, err := BuildProposalVariables(transcript, &schema.Glossary{}, 0, "")
vars, err := BuildProposalVariables(transcript, nil, 0, "")
if err != nil {
t.Fatal(err)
}
generated, err := GenerateCandidates(context.Background(), Request{WorkingTranscript: transcript, ModuleKey: "grammar", ModuleInstance: "grammar", ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, Config: &loaded.Config, PromptID: prompts.PromptIDModuleGrammarProposal, PromptVersion: prompts.VersionV1, Variables: vars, LLMClient: client})
generated, err := GenerateCandidates(context.Background(), Request{WorkingTranscript: transcript, ModuleKey: "grammar", ModuleInstance: "grammar", ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, PromptID: prompts.PromptIDModuleGrammarProposal, Variables: vars, LLMClient: client})
if err != nil {
t.Fatal(err)
}
if calls != 2 || len(generated.Enriched) != 1 {
t.Fatalf("repair path: calls=%d, result=%+v", calls, generated)
}
cfg := loaded.Config
cfg.Thresholds.Grammar = .8
input := validators.Request{ModuleKey: "grammar", Config: &cfg, CandidateProposal: generated.Enriched, WorkingTranscript: transcript}
input := validators.Request{ModuleKey: "grammar", Settings: config.Stage{ConfidenceThreshold: .8}, CandidateProposal: generated.Enriched, WorkingTranscript: transcript}
low, err := (validators.ConfidenceThresholdValidator{}).Validate(context.Background(), input)
if err != nil {
t.Fatal(err)

15
internal/framework/proposal_generation/generate_test.go
View File

@@ -10,8 +10,6 @@ import (
"strings"
"testing"
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
@@ -21,6 +19,7 @@ import (
)
type fakeStructuredClient struct {
secrets []string
responses []StructuredCorrectionSet
err error
completion contracts.StructuredCompletionResponse
@@ -51,6 +50,8 @@ func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contr
return response, nil
}
func (f *fakeStructuredClient) ConfiguredSecrets(context.Context) []string { return f.secrets }
type captureDiagnosticsWriter struct {
err error
lastStage string
@@ -69,14 +70,11 @@ func (w *captureDiagnosticsWriter) WriteInteraction(stage string, requestMetadat
func defaultRequest(t *testing.T) Request {
t.Helper()
cfg := config.Config{}
return Request{
ModuleKey: "test_module",
ModuleInstance: "test_module",
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
Config: &cfg,
PromptID: "modules.grammar.proposal",
PromptVersion: "v1",
Variables: map[string]string{"SectionJSON": "[]"},
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 7, Text: "teh"}, {ID: 25, Text: "other"}}},
StartIndex: 0,
@@ -323,13 +321,8 @@ func TestGenerateCandidatesDiagnosticsWrittenAndRedacted(t *testing.T) {
}
t.Setenv("PRIMARY_SECRET", primarySecret)
t.Setenv("VALIDATION_SECRET", validationSecret)
loaded, loadErr := config.Load(ck.Bytes("config.yml", []byte("version: 2\nbackends:\n primary:\n endpoint: https://primary.invalid/v1\n credential: PRIMARY_SECRET\n validation:\n endpoint: https://validation.invalid/v1\n credential: VALIDATION_SECRET\nllm:\n proposal_profile: base\n")), "")
if loadErr != nil {
t.Fatal(loadErr)
}
cfg := loaded.Config
client.secrets = []string{primarySecret, validationSecret}
req := defaultRequest(t)
req.Config = &cfg
req.LLMClient = client
req.DiagnosticsDir = t.TempDir()
req.Variables = map[string]string{"SectionJSON": "include secret " + primarySecret}

24
internal/framework/proposal_generation/module_proposal_test.go
View File

@@ -4,9 +4,11 @@ import (
"context"
"encoding/json"
"errors"
"reflect"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
domain "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
@@ -22,20 +24,20 @@ func TestExecuteModuleProposalBuildsMessagesFromSectionAndDescription(t *testing
},
}
section := contracts.SectionMetadata{Index: 7}
cfg := config.Config{}
cfg := config.Stage{}
cfg.Context.Description = "Hearing transcript with role titles."
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "teh"}}}
glossary := &schema.Glossary{Entries: []schema.GlossaryEntry{{Name: "X"}}}
glossary := []domain.Entry{{ID: "x", Name: "X"}}
var gotSectionIndex int
var gotDescription string
var gotTranscript *schema.Transcript
var gotGlossary *schema.Glossary
var gotGlossary []domain.Entry
out, err := ExecuteModuleProposal(context.Background(), ModuleProposalRequest{
ProposalRequest: contracts.ProposalRequest{
ExecutionContext: contracts.ExecutionContext{
Config: &cfg,
Settings: cfg,
WorkingTranscript: transcript,
Glossary: glossary,
Section: &section,
@@ -48,7 +50,7 @@ func TestExecuteModuleProposalBuildsMessagesFromSectionAndDescription(t *testing
LLMClient: client,
},
PromptID: prompts.PromptIDModuleGrammarProposal,
BuildVariables: func(inTranscript *schema.Transcript, inGlossary *schema.Glossary, sectionIndex int, transcriptDescription string) (map[string]string, error) {
BuildVariables: func(inTranscript *schema.Transcript, inGlossary []domain.Entry, sectionIndex int, transcriptDescription string) (map[string]string, error) {
gotSectionIndex = sectionIndex
gotDescription = transcriptDescription
gotTranscript = inTranscript
@@ -68,8 +70,8 @@ func TestExecuteModuleProposalBuildsMessagesFromSectionAndDescription(t *testing
if gotTranscript != transcript {
t.Fatalf("expected shared transcript pointer")
}
if gotGlossary != glossary {
t.Fatalf("expected shared glossary pointer")
if !reflect.DeepEqual(gotGlossary, glossary) {
t.Fatalf("expected glossary entries")
}
if len(client.calls) != 1 || client.calls[0].PromptID != prompts.PromptIDModuleGrammarProposal || client.calls[0].Context.SectionIndex == nil || *client.calls[0].Context.SectionIndex != 7 {
t.Fatalf("unexpected stage name calls: %+v", client.calls)
@@ -85,7 +87,7 @@ func TestExecuteModuleProposalValidatesInputs(t *testing.T) {
}
_, err := ExecuteModuleProposal(context.Background(), ModuleProposalRequest{
BuildVariables: func(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) (map[string]string, error) {
BuildVariables: func(transcript *schema.Transcript, glossary []domain.Entry, sectionIndex int, transcriptDescription string) (map[string]string, error) {
return nil, nil
},
})
@@ -95,7 +97,7 @@ func TestExecuteModuleProposalValidatesInputs(t *testing.T) {
_, err = ExecuteModuleProposal(context.Background(), ModuleProposalRequest{
PromptID: "missing.prompt.id",
BuildVariables: func(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) (map[string]string, error) {
BuildVariables: func(transcript *schema.Transcript, glossary []domain.Entry, sectionIndex int, transcriptDescription string) (map[string]string, error) {
return map[string]string{"SectionJSON": "[]"}, nil
},
})
@@ -108,7 +110,7 @@ func TestExecuteModuleProposalPropagatesBuilderError(t *testing.T) {
wantErr := errors.New("builder failed")
_, err := ExecuteModuleProposal(context.Background(), ModuleProposalRequest{
PromptID: prompts.PromptIDModuleGlossaryProposal,
BuildVariables: func(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) (map[string]string, error) {
BuildVariables: func(transcript *schema.Transcript, glossary []domain.Entry, sectionIndex int, transcriptDescription string) (map[string]string, error) {
return nil, wantErr
},
})
@@ -128,7 +130,7 @@ func TestExecuteModuleProposalPreservesFatalOperation(t *testing.T) {
LLMClient: client,
},
PromptID: prompts.PromptIDModuleGrammarProposal,
BuildVariables: func(*schema.Transcript, *schema.Glossary, int, string) (map[string]string, error) {
BuildVariables: func(*schema.Transcript, []domain.Entry, int, string) (map[string]string, error) {
return map[string]string{"SectionJSON": "[]"}, nil
},
})

3
internal/framework/proposal_generation/variables_test.go
View File

@@ -4,12 +4,13 @@ import (
"strings"
"testing"
domain "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
)
func TestBuildProposalVariablesPreservesSectionGlossaryAndDescription(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "gestures"}}}
glossary := &schema.Glossary{Entries: []schema.GlossaryEntry{{Name: "Jesters"}}}
glossary := []domain.Entry{{ID: "jesters", Name: "Jesters"}}
variables, err := BuildProposalVariables(transcript, glossary, 0, " Fantasy names ")
if err != nil {
t.Fatal(err)

23
internal/framework/runner/cancellation_test.go
View File

@@ -10,7 +10,6 @@ import (
"testing"
"time"
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
@@ -21,7 +20,8 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
pk "gitea.maximumdirect.net/eric/promptkit"
ck "go.maximumdirect.net/configkit"
pk "go.maximumdirect.net/promptkit"
)
type contextValidator struct {
@@ -47,23 +47,20 @@ func (t *cancelTransport) RoundTrip(req *http.Request) (*http.Response, error) {
func TestValidatorFailureCancelsActiveAndPendingProviderWork(t *testing.T) {
dir := t.TempDir()
if err := os.Mkdir(filepath.Join(dir, "profiles"), 0700); err != nil {
t.Fatal(err)
}
main := filepath.Join(dir, "config.yml")
body := "version: 2\nbackends:\n shared:\n endpoint: http://provider.invalid/v1\n concurrency: 1\n queue_capacity: 1\nllm:\n proposal_profile: base\n transient_retries: 0\n repair_attempts: 0\n"
if err := os.WriteFile(main, []byte(body), 0600); err != nil {
if err := os.WriteFile(main, []byte("version: 3\n"), 0600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "profiles/base.yaml"), []byte("profiles:\n base:\n backend: shared\n model: local\n timeout_seconds: 30\n"), 0600); err != nil {
resources := "backends:\n - id: shared\n endpoint: http://provider.invalid/v1\n concurrency_limit: 1\n queue_capacity: 1\nprofiles:\n - id: base\n backend: shared\n model: local\n"
if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte(resources), 0600); err != nil {
t.Fatal(err)
}
loaded, err := config.Load(ck.File(main), "")
loaded, err := config.Load(ck.Input{Source: ck.File(main)}, "")
if err != nil {
t.Fatal(err)
}
transport := &cancelTransport{entered: make(chan struct{}, 3)}
runtime, err := llm.NewRuntime(context.Background(), loaded.Config, main, &http.Client{Transport: transport})
runtime, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
if err != nil {
t.Fatal(err)
}
@@ -75,7 +72,7 @@ func TestValidatorFailureCancelsActiveAndPendingProviderWork(t *testing.T) {
started <- struct{}{}
}
}
client, err := runtime.NewRoleClient("validation", hooks)
client, err := runtime.NewRoleClient("validation", "base", config.LLM{OperationTimeout: 30 * time.Second}, hooks)
if err != nil {
t.Fatal(err)
}
@@ -101,7 +98,7 @@ func TestValidatorFailureCancelsActiveAndPendingProviderWork(t *testing.T) {
}
var out map[string]any
_, err := client.CompleteStructured(ctx, structuredoutput.Request{
PromptID: prompts.PromptIDValidatorEditorialReview, PromptVersion: prompts.VersionV1,
PromptID: prompts.PromptIDValidatorEditorialReview,
Variables: map[string]string{"TranscriptDescriptionBlock": "", "PayloadJSON": "[]"}, Schema: responseschema.ValidatorDecisionSetKey,
Context: structuredoutput.DiagnosticContext{ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: req.SectionIndex},
}, &out)
@@ -120,7 +117,7 @@ func TestValidatorFailureCancelsActiveAndPendingProviderWork(t *testing.T) {
}
done := make(chan pipelineOutcome, 1)
go func() {
result, err := runModulePipeline(ctx, collectSectionProposalsInput{Module: module, Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar"}, Policy: module.policy, Config: &loaded.Config, Sections: sections})
result, err := runModulePipeline(ctx, collectSectionProposalsInput{Module: module, Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar"}, Policy: module.policy, Sections: sections})
done <- pipelineOutcome{result: result, err: err}
}()
select {

92
internal/framework/runner/runner_test.go
View File

@@ -11,8 +11,8 @@ import (
"testing"
"time"
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/modules"
@@ -717,10 +717,7 @@ func TestRunnerDoesNotMutateInputTranscript(t *testing.T) {
}
func TestRunnerRepeatedModuleInstanceNames(t *testing.T) {
specs, err := contracts.ResolveModuleRunSpecs([]string{"glossary", "glossary"})
if err != nil {
t.Fatalf("ResolveModuleRunSpecs error: %v", err)
}
specs := contracts.ResolveModuleRunSpecs(&config.Config{Pipeline: config.Pipeline{Stages: []config.Stage{{Module: "glossary"}, {Module: "glossary"}}}})
if specs[0].InstanceName != "glossary_1" || specs[1].InstanceName != "glossary_2" {
t.Fatalf("unexpected instance names: %+v", specs)
}
@@ -920,10 +917,13 @@ func ptrConfig(c config.Config) *config.Config { return &c }
type fakeStructuredClient struct {
responses []validators.LLMValidationResponse
secrets []string
err error
calls int
}
func (f *fakeStructuredClient) ConfiguredSecrets(context.Context) []string { return f.secrets }
func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
_ = ctx
_ = req
@@ -955,10 +955,9 @@ func TestRunnerLLMValidatorApprovalApplied(t *testing.T) {
}})
cfg := runnerTestConfig()
out, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
ValidationLLMClient: client,
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}},
})
if err != nil {
t.Fatalf("Run error: %v", err)
@@ -978,10 +977,9 @@ func TestRunnerLLMValidatorRejectionPreventsApplication(t *testing.T) {
}})
cfg := runnerTestConfig()
out, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
ValidationLLMClient: client,
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}},
})
if err != nil {
t.Fatalf("Run error: %v", err)
@@ -1007,10 +1005,9 @@ func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress(
}})
cfg := runnerTestConfig()
out, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {ModuleKey: "m2", InstanceName: "m2"}},
ValidationLLMClient: client,
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {ModuleKey: "m2", InstanceName: "m2", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}},
})
if err != nil {
t.Fatalf("expected malformed validator response to downgrade, got %v", err)
@@ -1058,7 +1055,7 @@ func TestRunnerReconciledSurvivorsCompleteLaterValidation(t *testing.T) {
cfg := runnerTestConfig()
out, err := r.Run(context.Background(), RunInput{
Config: &cfg, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat bad"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, ValidationLLMClient: client,
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}},
})
if err != nil || out.FinalTranscript.Segments[0].Text != "the cat bad" || len(laterIndices) != 2 || laterIndices[0] != 0 || laterIndices[1] != 2 || client.calls != 1 {
t.Fatalf("survivor routing: text=%q later=%v calls=%d err=%v", out.FinalTranscript.Segments[0].Text, laterIndices, client.calls, err)
@@ -1083,10 +1080,9 @@ func TestRunnerLLMValidatorMissingDecisionRejectsBatch(t *testing.T) {
}})
cfg := runnerTestConfig()
_, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
ValidationLLMClient: client,
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}},
})
if err != nil {
t.Fatalf("expected missing decision downgrade, got %v", err)
@@ -1106,10 +1102,9 @@ func TestRunnerLLMValidatorDuplicateDecisionRejectsBatch(t *testing.T) {
}})
cfg := runnerTestConfig()
_, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
ValidationLLMClient: client,
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}},
})
if err != nil {
t.Fatalf("expected duplicate decision downgrade, got %v", err)
@@ -1139,10 +1134,9 @@ func TestRunnerLLMValidatorBatching(t *testing.T) {
}},
}})
out, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "alpha gamma"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
ValidationLLMClient: client,
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "alpha gamma"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}},
})
if err != nil {
t.Fatalf("unexpected run error: %v", err)
@@ -1161,13 +1155,8 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
validationSecret := "runner-validation-secret"
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: validationSecret}}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
t.Setenv("RUNNER_PRIMARY_KEY", primarySecret)
t.Setenv("RUNNER_VALIDATION_KEY", validationSecret)
loaded, loadErr := config.Load(ck.Bytes("config.yml", []byte("version: 2\nbackends:\n primary:\n endpoint: https://primary.invalid/v1\n credential: RUNNER_PRIMARY_KEY\n validation:\n endpoint: https://validation.invalid/v1\n credential: RUNNER_VALIDATION_KEY\nllm:\n proposal_profile: base\n")), "")
if loadErr != nil {
t.Fatal(loadErr)
}
cfg := loaded.Config
client.secrets = []string{primarySecret, validationSecret}
cfg := runnerTestConfig()
diagDir := t.TempDir()
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
@@ -1177,8 +1166,7 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
out, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple. " + primarySecret}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
ValidationLLMClient: client,
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}},
ValidationDiagnosticsDir: diagDir,
})
if err != nil {
@@ -1227,10 +1215,7 @@ func (m proposalGenerationModule) Propose(ctx context.Context, req contracts.Pro
ModuleInstance: req.RunSpec.InstanceName,
ReplacementPolicy: req.RunSpec.ReplacementPolicy,
WorkingTranscript: req.WorkingTranscript,
Config: req.Config,
Glossary: req.Glossary,
PromptID: "modules.grammar.proposal",
PromptVersion: "v1",
Variables: map[string]string{"TranscriptDescriptionBlock": "", "GlossaryJSON": "[]", "SectionJSON": "[]"},
LLMClient: req.LLMClient,
DiagnosticsDir: req.DiagnosticsDir,
@@ -1281,8 +1266,7 @@ func TestRunnerProposalGenerationHelperFlowsThroughPipeline(t *testing.T) {
out, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
ProposalLLMClient: client,
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ProposalClient: client, Settings: config.Stage{ConfidenceThreshold: cfg.Thresholds.Grammar}}},
ProposalDiagnosticsDir: diagDir,
})
if err != nil {
@@ -1315,13 +1299,11 @@ func TestRunnerGrammarModuleUsesConfidenceThreshold(t *testing.T) {
}
cfg := runnerTestConfig()
cfg.Thresholds.Grammar = 0.9
factory := modules.NewFactory(modules.Dependencies{})
factory := modules.NewFactory()
out, err := New(factory).Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello ,world"}}},
Glossary: &schema.Glossary{},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}},
ProposalLLMClient: client,
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello ,world"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", ProposalClient: client, Settings: config.Stage{ConfidenceThreshold: cfg.Thresholds.Grammar}}},
})
if err != nil {
t.Fatalf("Run error: %v", err)
@@ -1349,13 +1331,11 @@ func TestRunnerGlossaryModuleUsesConfidenceThreshold(t *testing.T) {
}
cfg := runnerTestConfig()
cfg.Thresholds.Glossary = 0.9
factory := modules.NewFactory(modules.Dependencies{})
factory := modules.NewFactory()
out, err := New(factory).Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures"}}},
Glossary: &schema.Glossary{Entries: []schema.GlossaryEntry{{Name: "Jesters", Category: "faction", Summary: "Faction"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "glossary", InstanceName: "glossary"}},
ProposalLLMClient: client,
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "glossary", InstanceName: "glossary", Glossary: []glossary.Entry{{Name: "Jesters", Category: "faction", Description: "Faction"}}, ProposalClient: client, Settings: config.Stage{ConfidenceThreshold: cfg.Thresholds.Glossary}}},
})
if err != nil {
t.Fatalf("Run error: %v", err)

19
internal/framework/validators/llm_context_test.go
View File

@@ -6,6 +6,7 @@ import (
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
)
@@ -19,11 +20,11 @@ func TestValidationContextCarriesAdjacentEvidenceAndRelevantDefinitions(t *testi
{ID: 4, Text: "Elsewhere."},
}}
req.CandidateProposal[0].TargetSegmentID = 2
req.Glossary = &schema.Glossary{Entries: []schema.GlossaryEntry{
{Name: "Jesters", Summary: "A traveling troupe"},
{Name: "Lyra", Summary: "The host"},
{Name: "Unrelated", Summary: "Not mentioned"},
}}
req.Glossary = []glossary.Entry{
{ID: "jesters", Name: "Jesters", Description: "A traveling troupe"},
{ID: "lyra", Name: "Lyra", Description: "The host"},
{ID: "unrelated", Name: "Unrelated", Description: "Not mentioned"},
}
payload, immediate := BuildLLMValidationRequest("spoken_form_plausibility", LLMValidatorTypeSpokenFormPlausibility, req)
if len(immediate) != 0 || len(payload.Items) != 1 {
t.Fatalf("payload: %+v immediate: %+v", payload, immediate)
@@ -40,10 +41,10 @@ func TestValidationContextCarriesAdjacentEvidenceAndRelevantDefinitions(t *testi
if err := json.Unmarshal([]byte(variables["PayloadJSON"]), &rendered); err != nil {
t.Fatal(err)
}
if strings.Contains(variables["PayloadJSON"], `"id":`) || rendered[0].CorrectionIndex != 1 || payload.Items[0].CorrectionIndex != 7 {
if strings.Contains(variables["PayloadJSON"], `"segment_id":`) || rendered[0].CorrectionIndex != 1 || payload.Items[0].CorrectionIndex != 7 {
t.Fatalf("leaked internal identity or mutated request: %s", variables["PayloadJSON"])
}
if len(rendered[0].NeighboringSegments) != 2 || rendered[0].GlossaryEntries[0].Summary != "A traveling troupe" {
if len(rendered[0].NeighboringSegments) != 2 || rendered[0].GlossaryEntries[0].Description != "A traveling troupe" {
t.Fatalf("rendered context: %+v", rendered)
}
// The context-free endpoint must not reach beyond the current chunk.
@@ -64,7 +65,7 @@ func TestValidationContextFitsBudgetWithoutDiscardingCandidate(t *testing.T) {
}
enriched := base
enriched.NeighboringSegments = []NeighboringSegment{{Position: "following", Text: strings.Repeat("long context ", 300)}}
enriched.GlossaryEntries = []schema.GlossaryEntry{{Name: "cat", Summary: strings.Repeat("long definition ", 300)}}
enriched.GlossaryEntries = []glossary.Entry{{Name: "cat", Description: strings.Repeat("long definition ", 300)}}
fitted := fitValidationContext(enriched, required, estimator)
tokens, err := estimateBatchTokens(estimator, []LLMValidationItem{fitted})
if err != nil || tokens > required || fitted.OriginalSegmentText != base.OriginalSegmentText || fitted.CorrectedSegmentText != base.CorrectedSegmentText {
@@ -80,7 +81,7 @@ func TestValidationContextFitsBudgetWithoutDiscardingCandidate(t *testing.T) {
func TestValidationContextSelectsGlossaryForEachReplacement(t *testing.T) {
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(7, "gestures", "Jesters"), mk(11, "gestures", "Actors")})
req.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Those gestures are back."}}}
req.Glossary = &schema.Glossary{Entries: []schema.GlossaryEntry{{Name: "Actors"}, {Name: "Jesters"}}}
req.Glossary = []glossary.Entry{{ID: "actors", Name: "Actors"}, {ID: "jesters", Name: "Jesters"}}
payload, immediate := BuildLLMValidationRequest("editorial_review", LLMValidatorTypeEditorialReview, req)
if len(immediate) != 0 || len(payload.Items) != 2 {
t.Fatalf("payload: %+v immediate: %+v", payload, immediate)

41
internal/framework/validators/llm_integration_test.go
View File

@@ -4,19 +4,24 @@ import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
ck "go.maximumdirect.net/configkit"
)
type validationTransport func(*http.Request) (*http.Response, error)
func (f validationTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) {
valid := `{"validations":[{"correction_index":1,"approved":true,"confidence":0.9,"reason":"ok"}]}`
for _, tc := range []struct {
@@ -32,41 +37,38 @@ func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) {
} {
t.Run(tc.name, func(t *testing.T) {
calls := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
transport := validationTransport(func(r *http.Request) (*http.Response, error) {
calls++
output := tc.responses[len(tc.responses)-1]
if calls <= len(tc.responses) {
output = tc.responses[calls-1]
}
if output == "MALFORMED_ENVELOPE" {
_, _ = fmt.Fprint(w, `{"choices":[]}`)
return
body := `{"choices":[]}`
if output != "MALFORMED_ENVELOPE" {
content, _ := json.Marshal(output)
body = fmt.Sprintf(`{"choices":[{"message":{"content":%s}}]}`, content)
}
content, _ := json.Marshal(output)
_, _ = fmt.Fprintf(w, `{"choices":[{"message":{"content":%s}}]}`, content)
}))
defer server.Close()
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
})
dir := t.TempDir()
if err := os.Mkdir(filepath.Join(dir, "profiles"), 0700); err != nil {
t.Fatal(err)
}
main := filepath.Join(dir, "config.yml")
content := fmt.Sprintf("version: 2\nbackends:\n offline:\n endpoint: %s/v1\nllm:\n proposal_profile: base\n transient_retries: 0\n repair_attempts: 1\n", server.URL)
if err := os.WriteFile(main, []byte(content), 0600); err != nil {
if err := os.WriteFile(main, []byte("version: 3\n"), 0600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "profiles", "base.yaml"), []byte("profiles:\n base:\n backend: offline\n model: local-model\n"), 0600); err != nil {
if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte("profiles:\n - {id: base, endpoint: 'https://example.invalid/v1', model: local-model, api_key_required: false}\n"), 0600); err != nil {
t.Fatal(err)
}
loaded, err := config.Load(ck.File(main), "")
loaded, err := config.Load(ck.Input{Source: ck.File(main)}, "")
if err != nil {
t.Fatal(err)
}
runtime, err := llm.NewRuntime(context.Background(), loaded.Config, main, server.Client())
runtime, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
if err != nil {
t.Fatal(err)
}
client, err := runtime.NewRoleClient("validation", llm.CallHooks{})
settings := loaded.Config.LLM
settings.TransientRetries, settings.RepairAttempts = 0, 1
client, err := runtime.NewRoleClient("validation", "base", settings, llm.CallHooks{})
if err != nil {
t.Fatal(err)
}
@@ -75,7 +77,6 @@ func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) {
t.Fatal(err)
}
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.Config = &loaded.Config
req.LLMClient = client
result, err := validator.Validate(context.Background(), req)
if calls != tc.wantCalls {

6
internal/framework/validators/llm_partial_test.go
View File

@@ -55,7 +55,7 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
validator.SetTokenEstimator(perItemEstimator{})
req := makeReq(candidates)
req.Config.Validation.MaxPromptTokens = 150
req.Settings.Validation.MaxPromptTokens = 150
req.LLMClient = client
result, err := validator.Validate(context.Background(), req)
if err != nil {
@@ -79,7 +79,7 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
nextValidator, _ := NewLLMBackedValidator("meaning_reversal_review", LLMValidatorTypeMeaningReversal)
nextValidator.SetTokenEstimator(perItemEstimator{})
nextReq := makeReq(survivors)
nextReq.Config.Validation.MaxPromptTokens = 150
nextReq.Settings.Validation.MaxPromptTokens = 150
nextReq.LLMClient = nextClient
next, err := nextValidator.Validate(context.Background(), nextReq)
if err != nil || len(next.Decisions) != 2 || len(nextClient.calls) != 2 || *nextClient.calls[0].Context.BatchIndex != 0 || *nextClient.calls[1].Context.BatchIndex != 1 {
@@ -144,7 +144,7 @@ func TestValidationFatalFailureKeepsEarlierFactsAndStops(t *testing.T) {
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
validator.SetTokenEstimator(perItemEstimator{})
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters"), mk(1, "gestures", "actors"), mk(2, "gestures", "figures")})
req.Config.Validation.MaxPromptTokens = 150
req.Settings.Validation.MaxPromptTokens = 150
req.LLMClient = client
result, err := validator.Validate(context.Background(), req)
if !errors.Is(err, cause) || len(result.Decisions) != 1 || len(result.Operations) != 2 || len(client.calls) != 2 {

13
internal/framework/validators/llm_validators_test.go
View File

@@ -59,15 +59,14 @@ func (f *fakeStructuredLLMClient) CompleteStructured(ctx context.Context, req St
}
func makeReq(candidates []proposals.EnrichedCorrectionProposal) Request {
cfg := config.Config{}
cfg.Validation.MaxPromptTokens = 10000
settings := config.Stage{Validation: config.Validation{MaxPromptTokens: 10000}}
return Request{
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple.", Categories: []string{"narration"}}}},
CandidateProposal: candidates,
ModuleKey: "homophones",
ModuleInstance: "homophones",
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
Config: &cfg,
Settings: settings,
}
}
@@ -314,9 +313,7 @@ func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *tes
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
}
req.LLMClient = client
cfg := config.Config{}
cfg.Validation.MaxPromptTokens = 200
req.Config = &cfg
req.Settings.Validation.MaxPromptTokens = 200
res, err := v.Validate(context.Background(), req)
if err != nil {
@@ -384,7 +381,7 @@ func TestEveryValidationTypeSelectsEmbeddedPromptAndVariables(t *testing.T) {
}
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = client
req.Config.Context.Description = " Fantasy scene "
req.Settings.Context.Description = " Fantasy scene "
if _, err := validator.Validate(context.Background(), req); err != nil {
t.Fatal(err)
}
@@ -392,7 +389,7 @@ func TestEveryValidationTypeSelectsEmbeddedPromptAndVariables(t *testing.T) {
t.Fatalf("calls=%d", len(client.calls))
}
call := client.calls[0]
if call.PromptID != tc.prompt || call.PromptVersion != prompts.VersionV1 || call.Schema != responseschema.ValidatorDecisionSetKey || call.Context.Role != "validation" || call.Context.BatchIndex == nil || !strings.Contains(call.Variables["PayloadJSON"], "correction_index") || !strings.Contains(call.Variables["TranscriptDescriptionBlock"], "Fantasy scene") {
if call.PromptID != tc.prompt || call.Schema != responseschema.ValidatorDecisionSetKey || call.Context.Role != "validation" || call.Context.BatchIndex == nil || !strings.Contains(call.Variables["PayloadJSON"], "correction_index") || !strings.Contains(call.Variables["TranscriptDescriptionBlock"], "Fantasy scene") {
t.Fatalf("mapping: %+v", call)
}
})

46
internal/framework/validators/protected_terms_test.go
View File

@@ -2,40 +2,36 @@ package validators
import (
"reflect"
"sort"
"testing"
domain "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
)
func TestExtractProtectedTermsStableDeduplicatedWithPlurals(t *testing.T) {
glossary := &schema.Glossary{
Entries: []schema.GlossaryEntry{
{Name: "Jesters", Aliases: []string{"Jester", " "}, Plural: "Jesters", Category: "faction", Summary: "Faction"},
{Name: "Hrank", Aliases: []string{"hrank", "Hrank"}, Plural: "Hranks", Category: "pc", Summary: "Character"},
{Name: "Lyra", Aliases: []string{}, Plural: "", Category: "npc", Summary: "NPC"},
},
func TestRecognizedTermsDeduplicateAndIncludePlurals(t *testing.T) {
plural := "Hranks"
entries := []domain.Entry{{ID: "jesters", Name: "Jesters", Aliases: []string{"Jester", " "}}, {ID: "hrank", Name: "Hrank", Aliases: []string{"hrank", "Hrank"}, Plural: &plural}, {ID: "lyra", Name: "Lyra"}}
var got []string
for _, term := range domain.Vocabulary(entries, nil) {
got = append(got, term.Spelling)
}
got := ExtractProtectedTerms(glossary)
sort.Strings(got)
want := []string{"Hrank", "Hranks", "Jester", "Jesters", "Jesterss", "Lyra", "Lyras"}
if !reflect.DeepEqual(got, want) {
t.Fatalf("unexpected protected terms\n got: %v\nwant: %v", got, want)
t.Fatalf("got %v want %v", got, want)
}
}
func TestExtractProtectedTermsEmptyGlossary(t *testing.T) {
if terms := ExtractProtectedTerms(&schema.Glossary{}); len(terms) != 0 {
t.Fatalf("expected no terms, got %v", terms)
if len(domain.Vocabulary(nil, nil)) != 0 {
t.Fatal("empty input produced terms")
}
}
func TestValidateProtectedTermsAppliesToNonGlossaryModule(t *testing.T) {
req := Request{
ModuleKey: "grammar",
Glossary: &schema.Glossary{
Entries: []schema.GlossaryEntry{{Name: "Jesters", Category: "faction", Summary: "Faction"}},
},
ModuleKey: "grammar",
Glossary: []domain.Entry{{ID: "jesters", Name: "Jesters", Category: "faction"}},
ProtectedTerms: domain.Vocabulary([]domain.Entry{{ID: "jesters", Name: "Jesters"}}, nil),
CandidateProposal: []proposals.EnrichedCorrectionProposal{
{
CorrectionProposal: proposals.CorrectionProposal{
@@ -52,7 +48,7 @@ func TestValidateProtectedTermsAppliesToNonGlossaryModule(t *testing.T) {
},
},
}
res, err := ValidateProtectedTerms(req, false)
res, err := ValidateProtectedTerms(req)
if err != nil {
t.Fatalf("validator error: %v", err)
}
@@ -64,9 +60,7 @@ func TestValidateProtectedTermsAppliesToNonGlossaryModule(t *testing.T) {
func TestValidateProtectedTermsGlossaryStageAllowsProtectedTermSwap(t *testing.T) {
req := Request{
ModuleKey: "glossary",
Glossary: &schema.Glossary{
Entries: []schema.GlossaryEntry{{Name: "Jesters", Category: "faction", Summary: "Faction"}},
},
Glossary: []domain.Entry{{ID: "jesters", Name: "Jesters", Category: "faction"}},
CandidateProposal: []proposals.EnrichedCorrectionProposal{
{
CorrectionProposal: proposals.CorrectionProposal{
@@ -79,7 +73,7 @@ func TestValidateProtectedTermsGlossaryStageAllowsProtectedTermSwap(t *testing.T
},
},
}
res, err := ValidateProtectedTerms(req, true)
res, err := ValidateProtectedTerms(req)
if err != nil {
t.Fatalf("validator error: %v", err)
}
@@ -103,10 +97,10 @@ func TestProtectedNameRepetitionRequiresSurvivingIdentityAndSpokenWordScope(t *t
t.Run(tc.name, func(t *testing.T) {
req := Request{ModuleKey: tc.module, ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: tc.before}}},
Glossary: &schema.Glossary{Entries: []schema.GlossaryEntry{{Name: "Jesters"}, {Name: "Lyra"}}},
ProtectedTerms: domain.Vocabulary([]domain.Entry{{ID: "jesters", Name: "Jesters"}, {ID: "lyra", Name: "Lyra"}}, nil),
CandidateProposal: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: tc.original, CorrectedText: tc.replacement, Confidence: 1}}},
}
result, err := ValidateProtectedTerms(req, false)
result, err := ValidateProtectedTerms(req)
if err != nil || len(result.Decisions) != 1 || result.Decisions[0].Approved != tc.allowed {
t.Fatalf("result=%+v err=%v", result, err)
}

14
internal/framework/validators/validators_test.go
View File

@@ -6,6 +6,7 @@ import (
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
domain "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
)
@@ -50,9 +51,7 @@ func TestEnforceDecisionCardinalityUnknownDecision(t *testing.T) {
}
func TestConfidenceThresholdValidator(t *testing.T) {
cfg := config.Config{}
cfg.Thresholds.Grammar = 0.8
req := Request{ModuleKey: "grammar", Config: &cfg, CandidateProposal: []proposals.EnrichedCorrectionProposal{
req := Request{ModuleKey: "grammar", Settings: config.Stage{ConfidenceThreshold: 0.8}, CandidateProposal: []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "teh", "the", 0.9),
mkCandidate(1, 1, "recieve", "receive", 0.7),
}}
@@ -161,16 +160,17 @@ func TestNoEffectValidator(t *testing.T) {
}
func TestValidateProtectedTerms(t *testing.T) {
glossary := &schema.Glossary{Entries: []schema.GlossaryEntry{{Name: "OpenAI", Aliases: []string{"Open AI"}, Plural: "OpenAIs", Category: "brand", Summary: "brand"}}}
entries := []domain.Entry{{ID: "openai", Name: "OpenAI", Aliases: []string{"Open AI"}, Category: "brand", Description: "brand"}}
req := Request{
Glossary: glossary,
ModuleKey: "grammar",
Glossary: entries,
ProtectedTerms: domain.Vocabulary(entries, nil),
ModuleKey: "grammar",
CandidateProposal: []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "OpenAI", "Open A I", 0.9),
mkCandidate(1, 1, "teh", "the", 0.9),
},
}
res, err := ValidateProtectedTerms(req, false)
res, err := ValidateProtectedTerms(req)
if err != nil {
t.Fatalf("Validate error: %v", err)
}