eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Complete assembled validation and migrate remaining tests

Browse Source
This commit is contained in:
Eric Rakestraw
2026-11-10 18:13:12 +00:00
parent ba1d5d2367
commit 2844ce7e6a
44 changed files with 577 additions and 1624 deletions
Download Patch File Download Diff File Expand all files Collapse all files

200
cmd/audita/main_integration_test.go
View File

@@ -233,15 +233,10 @@ func newProcessFixture(t *testing.T, mode, modules string) processFixture {
p := &fixtureProvider{mode: mode, secret: `selected-secret-"\-123`, entered: make(chan struct{}, 1)}
server := httptest.NewServer(http.HandlerFunc(p.serve))
t.Cleanup(server.Close)
if err := os.Mkdir(filepath.Join(dir, "profiles"), 0700); err != nil {
resources := fmt.Sprintf("backends:\n - id: local\n endpoint: %s/v1\n api_key_env: AUDITA_TEST_PROVIDER_KEY\nprofiles:\n - id: proposal\n backend: local\n model: local\n - id: validation\n backend: local\n model: local\n", server.URL)
if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte(resources), 0600); err != nil {
t.Fatal(err)
}
for _, id := range []string{"proposal", "validation"} {
profile := fmt.Sprintf("profiles:\n %s:\n backend: local\n model: local\n timeout_seconds: 5\n", id)
if err := os.WriteFile(filepath.Join(dir, "profiles", id+".yaml"), []byte(profile), 0600); err != nil {
t.Fatal(err)
}
}
configPath := filepath.Join(dir, "config.yml")
repairs := 0
if mode == "repair-filter-later-failure" || mode == "invalid-decision-after-repair" {
@@ -250,7 +245,17 @@ func newProcessFixture(t *testing.T, mode, modules string) processFixture {
if strings.HasPrefix(mode, "span-") || strings.HasPrefix(mode, "decision-") {
repairs = 1
}
configText := fmt.Sprintf("version: 2\nbackends:\n local:\n endpoint: %s/v1\n credential: AUDITA_TEST_PROVIDER_KEY\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n repair_attempts: %d\npipeline:\n modules: %s\ndiagnostics:\n work_dir: %s\n", server.URL, repairs, modules, filepath.Join(dir, "runs"))
var stageList []string
for _, name := range strings.Split(strings.Trim(modules, "[]"), ",") {
if name = strings.TrimSpace(name); name != "" {
stageList = append(stageList, " - module: "+name)
}
}
pipeline := " stages: []\n"
if len(stageList) != 0 {
pipeline = " stages:\n" + strings.Join(stageList, "\n") + "\n"
}
configText := fmt.Sprintf("version: 3\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n repair_attempts: %d\npipeline:\n%sdiagnostics:\n work_dir: %s\n", repairs, pipeline, filepath.Join(dir, "runs"))
if strings.HasPrefix(mode, "span-") || strings.HasPrefix(mode, "decision-") {
configText += "validation:\n max_prompt_tokens: 4096\nchunking:\n target_sections: 1\n max_section_tokens: 8192\n min_section_tokens: 1\nnormalization:\n max_segment_gap: 0\n"
}
@@ -271,7 +276,7 @@ func newProcessFixture(t *testing.T, mode, modules string) processFixture {
func (f processFixture) run(t *testing.T, includeOutput bool, extra ...string) (int, string, string) {
t.Helper()
args := []string{"-test.run=^TestProcessSubprocessHelper$", "--", "process", f.transcript, "--config", f.config, "--glossary", f.glossary}
args := []string{"-test.run=^TestProcessSubprocessHelper$", "--", "process", f.transcript, "--config", f.config}
if includeOutput {
args = append(args, "--output", f.output)
}
@@ -775,7 +780,7 @@ func TestSubprocessRespectsNegativeMeaningReviewOnDeletion(t *testing.T) {
func TestSubprocessDefaultFileOutputAndReports(t *testing.T) {
f := newProcessFixture(t, "normal", "[grammar]")
code, stdout, stderr := f.run(t, true)
if code != 0 || stdout != "" || !strings.HasPrefix(stderr, "Completed: 1 corrections applied; 0 work units with errors; 0 recoveries.") {
if code != 0 || stdout != "" || stderr != "" {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
output, err := os.ReadFile(f.output)
@@ -807,7 +812,7 @@ func TestSubprocessStdoutAndSelectedReportDirectory(t *testing.T) {
t.Fatal(err)
}
code, stdout, stderr := f.run(t, false, "--report-dir", reports)
if code != 0 || !strings.Contains(stdout, "Hello world") || !strings.HasPrefix(stderr, "Completed:") {
if code != 0 || !strings.Contains(stdout, "Hello world") || stderr != "" {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
if _, err := os.Stat(f.output); !os.IsNotExist(err) {
@@ -830,7 +835,7 @@ func TestSubprocessUsesIndependentTranscriptReportAndDebugDestinations(t *testin
}
f.output = filepath.Join(transcriptDir, "corrected.json")
code, stdout, stderr := f.run(t, true, "--report-dir", reportDir, "--debug", "--work-dir", debugDir)
if code != 0 || stdout != "" || !strings.HasPrefix(stderr, "Completed:") {
if code != 0 || stdout != "" || !strings.Contains(stderr, "Completed:") {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
if output, err := os.ReadFile(f.output); err != nil || !bytes.Contains(output, []byte("Hello world")) {
@@ -859,42 +864,34 @@ func TestSubprocessReportingAndDebugSelections(t *testing.T) {
{false, false}, {false, true}, {true, false}, {true, true},
} {
f := newProcessFixture(t, "normal", "[grammar]")
if selection.reporting && selection.debug {
original, err := os.ReadFile(f.config)
if err != nil {
t.Fatal(err)
}
old := fmt.Sprintf("backends:\n local:\n endpoint: %s/v1\n credential: AUDITA_TEST_PROVIDER_KEY", f.server.URL)
inherited := fmt.Sprintf("backends:\n local-base:\n endpoint: %s/v1\n credential: AUDITA_TEST_PROVIDER_KEY\n concurrency: 2\n queue_capacity: 0\n local:\n extends: local-base\n concurrency: 1", f.server.URL)
updated := strings.Replace(string(original), old, inherited, 1)
if updated == string(original) {
t.Fatal("process fixture backend configuration was not found")
}
if err := os.WriteFile(f.config, []byte(updated), 0600); err != nil {
t.Fatal(err)
}
}
reportDir := filepath.Join(f.dir, "missing")
extra := []string{"--reporting=false", "--report-dir", reportDir}
extra := []string{"--verbose", "--reporting=false", "--report-dir", reportDir}
if selection.reporting {
reportDir = filepath.Join(f.dir, "selected-reports")
if err := os.Mkdir(reportDir, 0700); err != nil {
t.Fatal(err)
}
extra = []string{"--reporting=true", "--report-dir", reportDir}
extra = []string{"--verbose", "--reporting=true", "--report-dir", reportDir}
}
if selection.debug {
extra = append(extra, "--debug")
}
code, stdout, stderr := f.run(t, true, extra...)
if code != 0 || stdout != "" || !strings.HasPrefix(stderr, "Completed:") {
if code != 0 || stdout != "" || !strings.Contains(stderr, "Completed:") {
t.Fatalf("selection=%+v code=%d stdout=%q stderr=%q", selection, code, stdout, stderr)
}
output, err := os.ReadFile(f.output)
if err != nil {
t.Fatal(err)
}
summary := strings.SplitN(stderr, "\n", 2)[0]
summary := ""
for _, line := range strings.Split(stderr, "\n") {
if strings.HasPrefix(line, "Completed:") {
summary = line
break
}
}
if expectedOutput == "" {
expectedOutput, expectedSummary = string(output), summary
} else if string(output) != expectedOutput || summary != expectedSummary {
@@ -916,31 +913,7 @@ func TestSubprocessReportingAndDebugSelections(t *testing.T) {
t.Fatal(err)
}
}
if selection.reporting {
effective, err := os.ReadFile(filepath.Join(dirs[0], "effective-config.redacted.yaml"))
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(effective), f.server.URL+"/v1") || !strings.Contains(string(effective), "concurrency: 1") || !strings.Contains(string(effective), "queue_capacity: 0") || strings.Contains(string(effective), f.provider.secret) {
t.Fatalf("effective inherited settings: %s", effective)
}
origins, err := os.ReadFile(filepath.Join(dirs[0], "config-origins.yaml"))
if err != nil {
t.Fatal(err)
}
for _, want := range []string{"from: /backends/local-base/endpoint", "to: /backends/local/endpoint", "reference: /backends/local/extends"} {
if !strings.Contains(string(origins), want) {
t.Fatalf("configuration origins missing %q: %s", want, origins)
}
}
if strings.Contains(string(origins), f.provider.secret) {
t.Fatal("configuration origins exposed the resolved inherited credential")
}
code, explain, stderr := f.runConfigView(t, "explain")
if code != 0 || stderr != "" || !strings.Contains(explain, "from: /backends/local-base/endpoint") || strings.Contains(explain, f.provider.secret) {
t.Fatalf("inheritance explanation: code=%d stdout=%s stderr=%s", code, explain, stderr)
}
}
}
}
}
@@ -954,7 +927,7 @@ func boolInt(value bool) int {
func TestSubprocessAllModelFailureStillCompletes(t *testing.T) {
f := newProcessFixture(t, "all-model-failure", "[grammar, grammar]")
code, stdout, stderr := f.run(t, true)
code, stdout, stderr := f.run(t, true, "--verbose")
if code != 0 || stdout != "" || !strings.Contains(stderr, "2 work units with errors") {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
@@ -973,7 +946,7 @@ func TestSubprocessAllModelFailureStillCompletes(t *testing.T) {
func TestSubprocessAllValidationFailureStillCompletes(t *testing.T) {
f := newProcessFixture(t, "all-validation-failure", "[grammar, grammar]")
code, stdout, stderr := f.run(t, true)
code, stdout, stderr := f.run(t, true, "--verbose")
if code != 0 || stdout != "" || !strings.Contains(stderr, "2 work units with errors") {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
@@ -989,7 +962,7 @@ func TestSubprocessAllValidationFailureStillCompletes(t *testing.T) {
func TestSubprocessRecoveredFilteringSurvivesLaterFailure(t *testing.T) {
f := newProcessFixture(t, "repair-filter-later-failure", "[grammar, grammar]")
code, _, stderr := f.run(t, true)
code, _, stderr := f.run(t, true, "--verbose")
if code != 0 || !strings.Contains(stderr, "1 corrections applied; 1 work units with errors; 1 recoveries") {
t.Fatalf("code=%d stderr=%q errors=%+v", code, stderr, f.errors(t))
}
@@ -1019,7 +992,7 @@ func TestSubprocessRecoveredFilteringSurvivesLaterFailure(t *testing.T) {
func TestSubprocessRepairedInvalidDecisionsRemainAnError(t *testing.T) {
f := newProcessFixture(t, "invalid-decision-after-repair", "[grammar]")
code, _, stderr := f.run(t, true)
code, _, stderr := f.run(t, true, "--verbose")
if code != 0 || !strings.Contains(stderr, "0 corrections applied; 1 work units with errors; 0 recoveries") {
t.Fatalf("code=%d stderr=%q", code, stderr)
}
@@ -1037,7 +1010,7 @@ func TestSubprocessCancellationLeavesFailedReportsWithoutTranscript(t *testing.T
f := newProcessFixture(t, "wait-cancel", "[grammar, grammar]")
ctx, stop := context.WithTimeout(context.Background(), 5*time.Second)
defer stop()
args := []string{"-test.run=^TestProcessSubprocessHelper$", "--", "process", f.transcript, "--config", f.config, "--glossary", f.glossary, "--output", f.output}
args := []string{"-test.run=^TestProcessSubprocessHelper$", "--", "process", f.transcript, "--config", f.config, "--output", f.output}
command := exec.CommandContext(ctx, os.Args[0], args...)
command.Env = append(os.Environ(), "AUDITA_HELPER_PROCESS=1", "AUDITA_TEST_PROVIDER_KEY="+f.provider.secret)
var stderr bytes.Buffer
@@ -1067,7 +1040,7 @@ func TestSubprocessCancellationLeavesFailedReportsWithoutTranscript(t *testing.T
func TestSubprocessPartialValidationAndLaterStage(t *testing.T) {
f := newProcessFixture(t, "partial-validation", "[grammar, grammar]")
code, _, stderr := f.run(t, true)
code, _, stderr := f.run(t, true, "--verbose")
if code != 0 || !strings.Contains(stderr, "1 corrections applied; 1 work units with errors") {
t.Fatalf("code=%d stderr=%q", code, stderr)
}
@@ -1081,55 +1054,41 @@ func TestSubprocessPartialValidationAndLaterStage(t *testing.T) {
}
}
func TestSubprocessInvalidInputLeavesFailedReports(t *testing.T) {
func TestSubprocessInvalidInputCreatesNoOutput(t *testing.T) {
f := newProcessFixture(t, "normal", "[grammar]")
if err := os.WriteFile(f.transcript, []byte("not JSON"), 0600); err != nil {
t.Fatal(err)
}
code, _, _ := f.run(t, true)
if code != 1 || f.provider.callCount() != 0 {
if code != 2 || f.provider.callCount() != 0 {
t.Fatalf("invalid input dispatched: code=%d calls=%d", code, f.provider.callCount())
}
if document := f.errors(t); document.Status != reporting.StatusFailed || len(document.Errors) != 1 || document.Errors[0].Scope != "run" {
t.Fatalf("failed report: %+v", document)
}
if _, err := os.Stat(f.output); !os.IsNotExist(err) {
t.Fatalf("failed transcript published: %v", err)
}
var recoveries reporting.RecoveriesDocument
if raw, err := os.ReadFile(filepath.Join(f.dir, "recoveries.yml")); err != nil {
t.Fatal(err)
} else if err := yaml.Unmarshal(raw, &recoveries); err != nil {
t.Fatal(err)
}
if recoveries.Status != reporting.StatusFailed {
t.Fatalf("recoveries status: %+v", recoveries)
}
if raw, err := os.ReadFile(filepath.Join(f.dir, "report.md")); err != nil || !bytes.Contains(raw, []byte("Status: failed")) {
t.Fatalf("Markdown status: %s %v", raw, err)
}
}
func TestSubprocessMissingInheritanceParentFailsBeforeProviderDispatch(t *testing.T) {
f := newProcessFixture(t, "normal", "[grammar]")
original, err := os.ReadFile(f.config)
original, err := os.ReadFile(filepath.Join(f.dir, "prompts.yml"))
if err != nil {
t.Fatal(err)
}
updated := strings.Replace(string(original), " local:\n endpoint:", " local:\n extends: missing\n endpoint:", 1)
updated := strings.Replace(string(original), " - id: local\n", " - id: local\n extends: {id: missing}\n", 1)
if updated == string(original) {
t.Fatal("process fixture backend configuration was not found")
}
if err := os.WriteFile(f.config, []byte(updated), 0600); err != nil {
if err := os.WriteFile(filepath.Join(f.dir, "prompts.yml"), []byte(updated), 0600); err != nil {
t.Fatal(err)
}
code, stdout, stderr := f.run(t, true)
if code != 2 || stdout != "" || f.provider.callCount() != 0 || !strings.Contains(stderr, "referenced sibling does not exist") {
if code != 2 || stdout != "" || f.provider.callCount() != 0 {
t.Fatalf("missing parent handling: code=%d stdout=%q stderr=%q provider calls=%d", code, stdout, stderr, f.provider.callCount())
}
}
func TestSubprocessInvalidInputWritesFailedReportsToSelectedDirectory(t *testing.T) {
func TestSubprocessInvalidInputDoesNotCreateReportsInSelectedDirectory(t *testing.T) {
f := newProcessFixture(t, "normal", "[grammar]")
if err := os.WriteFile(f.transcript, []byte("not JSON"), 0600); err != nil {
t.Fatal(err)
@@ -1139,25 +1098,13 @@ func TestSubprocessInvalidInputWritesFailedReportsToSelectedDirectory(t *testing
t.Fatal(err)
}
code, _, _ := f.run(t, true, "--report-dir", reportDir)
if code != 1 || f.provider.callCount() != 0 {
if code != 2 || f.provider.callCount() != 0 {
t.Fatalf("invalid input dispatched: code=%d calls=%d", code, f.provider.callCount())
}
raw, err := os.ReadFile(filepath.Join(reportDir, "errors.yml"))
if err != nil {
t.Fatal(err)
}
var document reporting.ErrorsDocument
if err := yaml.Unmarshal(raw, &document); err != nil {
t.Fatal(err)
}
if document.Status != reporting.StatusFailed || len(document.Errors) != 1 || document.Errors[0].Scope != "run" {
t.Fatalf("failed report: %+v", document)
}
if _, err := os.Stat(filepath.Join(reportDir, "recoveries.yml")); err != nil {
t.Fatal(err)
}
if raw, err := os.ReadFile(filepath.Join(reportDir, "report.md")); err != nil || !bytes.Contains(raw, []byte("Status: failed")) {
t.Fatalf("Markdown status: %s %v", raw, err)
for _, name := range []string{"errors.yml", "recoveries.yml", "report.md"} {
if _, err := os.Stat(filepath.Join(reportDir, name)); !os.IsNotExist(err) {
t.Fatalf("report created before preflight: %s %v", name, err)
}
}
if _, err := os.Stat(filepath.Join(f.dir, "errors.yml")); !os.IsNotExist(err) {
t.Fatalf("failed report appeared in default directory: %v", err)
@@ -1182,7 +1129,7 @@ func TestSubprocessOutputFailureBeforeModelDispatch(t *testing.T) {
func TestSubprocessOptionalDebugFailureDoesNotChangeOutcome(t *testing.T) {
f := newProcessFixture(t, "normal", "[grammar]")
code, _, stderr := f.run(t, true, "--debug", "--work-dir", f.transcript)
if code != 0 || !strings.HasPrefix(stderr, "Completed: 1 corrections applied") || !strings.Contains(stderr, "debug directory") {
if code != 0 || !strings.Contains(stderr, "Completed: 1 corrections applied") || !strings.Contains(stderr, "work directory") {
t.Fatalf("optional debug failure changed processing: code=%d stderr=%q", code, stderr)
}
if _, err := os.Stat(f.output); err != nil {
@@ -1197,7 +1144,7 @@ func TestSubprocessTranscriptPreflightWithoutReporting(t *testing.T) {
f := newProcessFixture(t, "normal", "[grammar]")
f.output = filepath.Join(f.dir, "missing", "output.json")
code, _, stderr := f.run(t, true, "--reporting=false")
if code == 0 || f.provider.callCount() != 0 || !strings.Contains(stderr, "transcript output parent") {
if code == 0 || f.provider.callCount() != 0 || !strings.Contains(stderr, "prepare transcript output") {
t.Fatalf("transcript preflight: code=%d calls=%d stderr=%q", code, f.provider.callCount(), stderr)
}
}
@@ -1256,12 +1203,12 @@ func TestSubprocessPathAnnouncementsAreRedacted(t *testing.T) {
selected, unused := "selected-path-secret", "unused-path-secret"
f.provider.secret = selected
t.Setenv("AUDITA_UNUSED_PATH_KEY", unused)
cfg, err := os.ReadFile(f.config)
cfg, err := os.ReadFile(filepath.Join(f.dir, "prompts.yml"))
if err != nil {
t.Fatal(err)
}
cfg = bytes.Replace(cfg, []byte("backends:\n"), []byte("backends:\n unused:\n endpoint: http://unused.invalid/v1\n credential: AUDITA_UNUSED_PATH_KEY\n"), 1)
if err := os.WriteFile(f.config, cfg, 0600); err != nil {
cfg = bytes.Replace(cfg, []byte("backends:\n"), []byte("backends:\n - id: unused\n endpoint: http://unused.invalid/v1\n api_key_env: AUDITA_UNUSED_PATH_KEY\n"), 1)
if err := os.WriteFile(filepath.Join(f.dir, "prompts.yml"), cfg, 0600); err != nil {
t.Fatal(err)
}
reportDir := filepath.Join(f.dir, unused)
@@ -1276,20 +1223,20 @@ func TestSubprocessPathAnnouncementsAreRedacted(t *testing.T) {
}
}
code, _, stderr := f.run(t, true, "--debug", "--work-dir", debugParent)
if (!fail && code != 0) || (fail && code != 1) {
if (!fail && code != 0) || (fail && code != 2) {
t.Fatalf("code=%d stderr=%q", code, stderr)
}
if strings.Contains(stderr, selected) || strings.Contains(stderr, unused) {
t.Fatalf("credential in stderr: %q", stderr)
}
if !strings.Contains(stderr, "[REDACTED]") {
if !fail && !strings.Contains(stderr, "[REDACTED]") {
t.Fatalf("missing redacted announcement: %q", stderr)
}
if _, err := os.Stat(filepath.Join(reportDir, "report.md")); err != nil {
if _, err := os.Stat(filepath.Join(reportDir, "report.md")); !fail && err != nil {
t.Fatal(err)
}
dirs, err := os.ReadDir(debugParent)
if err != nil || len(dirs) != 1 {
if !fail && (err != nil || len(dirs) != 1) {
t.Fatalf("debug location: %v %v", dirs, err)
}
if !fail {
@@ -1303,7 +1250,7 @@ func TestSubprocessPathAnnouncementsAreRedacted(t *testing.T) {
func TestSubprocessInteractionWriteFailureIsVisibleAndNonfatal(t *testing.T) {
f := newProcessFixture(t, "normal", "[grammar]")
code, _, baselineSummary := f.run(t, true)
code, _, baselineSummary := f.run(t, true, "--verbose")
if code != 0 {
t.Fatal(baselineSummary)
}
@@ -1318,16 +1265,16 @@ func TestSubprocessInteractionWriteFailureIsVisibleAndNonfatal(t *testing.T) {
t.Errorf("debug directory: %v %v", dirs, err)
return
}
if err := os.WriteFile(filepath.Join(dirs[0], "grammar"), []byte("block interaction writes"), 0600); err != nil {
if err := os.WriteFile(filepath.Join(dirs[0], "grammar_1"), []byte("block interaction writes"), 0600); err != nil {
t.Error(err)
}
}
f.provider.mu.Unlock()
code, _, stderr := f.run(t, true, "--debug")
if code != 0 || !strings.HasPrefix(stderr, strings.SplitN(baselineSummary, "\n", 2)[0]) {
if code != 0 || !strings.Contains(stderr, strings.SplitN(baselineSummary, "\n", 2)[0]) {
t.Fatalf("changed outcome: %d %q", code, stderr)
}
if !strings.Contains(stderr, "debug proposal interaction") || !strings.Contains(stderr, "debug validation interaction") {
if !strings.Contains(stderr, "proposal interaction") || !strings.Contains(stderr, "validation interaction") {
t.Fatalf("missing debug failures: %q", stderr)
}
output, err := os.ReadFile(f.output)
@@ -1360,7 +1307,7 @@ func TestSubprocessInPlaceTranscript(t *testing.T) {
t.Fatal(err)
}
if invalid {
if code != 1 || !bytes.Equal(before, after) {
if code != 2 || !bytes.Equal(before, after) {
t.Fatalf("failed processing changed input: code=%d %s", code, stderr)
}
} else if code != 0 || stdout != "" || !bytes.Contains(after, []byte("Hello world")) {
@@ -1374,15 +1321,18 @@ func TestSubprocessProfileSources(t *testing.T) {
f := newProcessFixture(t, "success", "[]")
// PromptKit uses profile IDs, not filename stems. A report-named profile is
// still an input and must not be overwritten during report initialization.
loaded := filepath.Join(f.dir, "profiles", "errors.yml")
if err := os.Rename(filepath.Join(f.dir, "profiles", "proposal.yaml"), loaded); err != nil {
if err := os.Mkdir(filepath.Join(f.dir, "prompts"), 0700); err != nil {
t.Fatal(err)
}
loaded := filepath.Join(f.dir, "prompts", "errors.yml")
if err := os.Rename(filepath.Join(f.dir, "prompts.yml"), loaded); err != nil {
t.Fatal(err)
}
before, err := os.ReadFile(loaded)
if err != nil {
t.Fatal(err)
}
f.output = filepath.Join(f.dir, "profiles", "proposal.yml")
f.output = filepath.Join(f.dir, "prompts", "proposal.yml")
code, _, stderr := f.run(t, true, "--reporting=false")
if code != 0 {
t.Fatalf("unused alternate rejected: %d %s", code, stderr)
@@ -1396,12 +1346,12 @@ func TestSubprocessProfileSources(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if code != 2 || !bytes.Equal(before, after) || f.provider.callCount() != 0 {
if code != 1 || !bytes.Equal(before, after) || f.provider.callCount() != 0 {
t.Fatalf("loaded profile not protected: %d %s", code, stderr)
}
parentDir := filepath.Join(f.dir, "profiles", "catalog")
childDir := filepath.Join(f.dir, "profiles", "nested", "child")
parentDir := filepath.Join(f.dir, "prompts", "catalog")
childDir := filepath.Join(f.dir, "prompts", "nested", "child")
for _, directory := range []string{parentDir, childDir} {
if err := os.MkdirAll(directory, 0700); err != nil {
t.Fatal(err)
@@ -1411,8 +1361,8 @@ func TestSubprocessProfileSources(t *testing.T) {
child := filepath.Join(childDir, "override.yaml")
resource := filepath.Join(parentDir, "report.md")
for path, content := range map[string]string{
parent: "prompts:\n source:\n release/1~draft:\n messages:\n - role: user\n content_file: ./report.md\n output:\n format: text\n validation_mode: basic\n",
child: "prompts:\n literal/~child:\n extends: source\n",
parent: "prompts:\n - id: source\n messages:\n - role: user\n content_file: report.md\n output:\n format: text\n validation_mode: none\n",
child: "prompts:\n - id: child\n extends: {id: source}\n",
resource: "Captured operator message.\n",
} {
if err := os.WriteFile(path, []byte(content), 0600); err != nil {
@@ -1429,7 +1379,7 @@ func TestSubprocessProfileSources(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if code != 2 || !bytes.Equal(resourceBefore, resourceAfter) || f.provider.callCount() != 0 {
if code != 1 || !bytes.Equal(resourceBefore, resourceAfter) || f.provider.callCount() != 0 {
t.Fatalf("consumed message file not protected as transcript output: %d %s", code, stderr)
}
@@ -1439,7 +1389,7 @@ func TestSubprocessProfileSources(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if code != 2 || !bytes.Equal(resourceBefore, resourceAfter) || f.provider.callCount() != 0 {
if code != 1 || !bytes.Equal(resourceBefore, resourceAfter) || f.provider.callCount() != 0 {
t.Fatalf("consumed message file not protected as report output: %d %s", code, stderr)
}
}

4
docs/roadmap/commandkit.md
View File

@@ -3,14 +3,14 @@
**Updated:** 2026-11-10
**Status:** Adoption target and design decisions resolved; the upstream release
gate is satisfied. Implementation remains pending. The
gate is satisfied and implementation is complete. The
[master roadmap](library-adoption.md) owns shared direction and exact dependency
revisions; the [implementation plan](implementation.md) owns sequencing. See the
[ConfigKit](configkit.md) and [PromptKit](promptkit.md) outlines for their contracts.
## Baseline and Target
Audita currently uses application-owned CLI parsing, inspection commands, and
Before this work, Audita used application-owned CLI parsing, inspection commands, and
reporting, described in the [CLI contract](../cli.md). Adopt CommandKit core v0.6.0
with ckconfig and ckprompt v0.7.0 as the standard operator interface. Use supported
command declarations, typed inputs, supplied-value tracking, help/version,

4
docs/roadmap/configkit.md
View File

@@ -2,13 +2,13 @@
**Updated:** 2026-11-10
**Status:** Target decisions resolved; implementation pending. The
**Status:** Target decisions resolved and implementation complete. The
[master roadmap](library-adoption.md) owns shared policy and dependency selection;
the [implementation plan](implementation.md) owns sequencing.
## Baseline and Target
Audita currently uses ConfigKit v0.7.1 through its former import path. Its
Before this work, Audita used ConfigKit v0.7.1 through its former import path. Its
[configuration contract](../config.md) describes application file discovery,
overlays, defaults, inheritance, secret references, and explicit CLI settings.

2
docs/roadmap/glossary-stages.md
View File

@@ -8,7 +8,7 @@ placement, optional default sources, collection-scoped entry IDs, extensible
entry content fields, and optional stage IDs. Active glossary vocabulary takes
precedence over overlapping protection. Empty collections are permitted, with
Audita preflight rejecting empty glossary stages. Design decisions are resolved;
implementation is pending. The
implementation is complete. The
[master roadmap](library-adoption.md) owns release completion; the
[implementation plan](implementation.md) owns sequencing.

6
docs/roadmap/implementation.md
View File

@@ -616,6 +616,12 @@ in the real repository during this sprint.
## Stage 15: Verify the assembled target and hand it off
**Completed.** The assembled candidate passed the selected ReleaseKit checks
with Go 1.27.3 in a clean disposable repository. A separate annotated-tag
fixture exercised the checked-in delivery workflow against a local mock,
including both Linux assets, checksums, and rejection of a mismatched commit.
Hosted delivery and source publication remain separate.
1. Run `go mod tidy` under the selected toolchain with `GOWORK=off`; inspect the
selected module graph, imports, tool registrations, and attribution together.
Confirm no old library identities, committed workspace files, or filesystem

6
docs/roadmap/library-adoption.md
View File

@@ -3,7 +3,7 @@
**Updated:** 2026-11-10
**Status:** Target decisions are recorded across this roadmap and its four child
roadmaps; implementation and dependency changes remain pending. The
roadmaps; the implementation and selected dependency changes are complete. The
[implementation plan](implementation.md) owns the sprint sequence.
## Objective and Document Ownership
@@ -116,8 +116,8 @@ the independent shared debug binding for capture and the resolved reporting-mode
accessor for quiet destination validation. The matching released integrations
were reviewed together. The [CommandKit outline](commandkit.md#adoption-gate-released-debug-binding-and-reporting-mode-access)
records the API contract and review evidence. The selected release set removes
the upstream availability blocker; Audita implementation and acceptance remain
pending under the [implementation plan](implementation.md).
the upstream availability blocker; Audita implementation and acceptance are
complete under the [implementation plan](implementation.md).
**Main-relative loading gate — satisfied:** ConfigKit v1.2.0 implements the
requested companion import binding and derived layered-overlay context. Follow

4
docs/roadmap/promptkit.md
View File

@@ -3,13 +3,13 @@
**Updated:** 2026-11-10
**Status:** Adoption direction and automatic operator-source selection resolved;
implementation remains pending. Follow the [master roadmap](library-adoption.md)
implementation is complete. Follow the [master roadmap](library-adoption.md)
for shared adoption principles and dependency identities, and the
[implementation plan](implementation.md) for sequencing.
## Baseline and Target
Audita currently uses PromptKit v0.14.0. Its
Before this work, Audita used PromptKit v0.14.0. Its
[LLM runtime](../internal/llm-runtime.md) uses a shared engine, application-owned
backend overrides, curated profile inspection, and prepared execution.

7
go.mod
View File

@@ -3,21 +3,17 @@ module gitea.maximumdirect.net/eric/audita/v3
go 1.27.0
require (
gitea.maximumdirect.net/eric/ConfigKit v0.7.1
gitea.maximumdirect.net/eric/promptkit v0.14.0
go.maximumdirect.net/ckconfig v0.7.0
go.maximumdirect.net/ckprompt v0.7.0
go.maximumdirect.net/commandkit v0.6.0
go.maximumdirect.net/configkit v1.2.0
go.maximumdirect.net/promptkit v0.21.0
go.maximumdirect.net/releasekit v0.9.0
golang.org/x/mod v0.40.0
gopkg.in/yaml.v3 v3.0.1
)
require (
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0 // indirect
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/rogpeppe/go-internal v1.14.1 // indirect
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect
@@ -25,6 +21,7 @@ require (
github.com/spf13/pflag v1.0.9 // indirect
github.com/yuin/goldmark/v2 v2.1.6 // indirect
go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
golang.org/x/mod v0.40.0 // indirect
golang.org/x/text v0.14.0 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
)

17
go.sum
View File

@@ -1,15 +1,4 @@
gitea.maximumdirect.net/eric/ConfigKit v0.7.1 h1:q3HrX4WWCPt1WfMydqFEaAmCf9s6Dq+vknJj0C/mdcI=
gitea.maximumdirect.net/eric/ConfigKit v0.7.1/go.mod h1:C82sbyKHYMV1OwaD2GJanFrwTfcZfjlW2JsGze5b0kA=
gitea.maximumdirect.net/eric/ReleaseKit v0.4.1 h1:Qn+mdebYOzCVCEXgCbuGSO/WHmJtItwaQ8e3XwY3qtI=
gitea.maximumdirect.net/eric/ReleaseKit v0.4.1/go.mod h1:LVodqNQ3OIKv9NBq+fFeyMByd1lHY25YOmmpah/NaNI=
gitea.maximumdirect.net/eric/promptkit v0.14.0 h1:HUzlcrFk18dNGPBgnHqjAw87O0c9Wl/IC1xc/VmzL10=
gitea.maximumdirect.net/eric/promptkit v0.14.0/go.mod h1:2YGvNca3bNXZP+e+pJeGzsNUcxwkHvzvuoc+LNNiT3E=
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0 h1:lzfCDbzJQRGzIlRPRprnAV9hVVL5zUQy80VMQ0kbYa0=
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0/go.mod h1:HjS0f+3q0IdHCTEz+VVAEgIwIWJFAZUjuKdAtvtqKdg=
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0 h1:h12ZTRldAmpLAomj0oQRGteFxHLN1LPE6XDl9zJNbm8=
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0/go.mod h1:OEFBeKdjTsLzqJwLmXI2O5gG3r64FktcVJVb3mBrRfg=
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
github.com/creack/pty v1.1.9 h1:uDmaGzcdjhF4i/plgjmEsriH11Y0o7RKapEf/LDaM3w=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
@@ -18,12 +7,10 @@ github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLf
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
@@ -53,12 +40,8 @@ go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs=
golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE=
golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI=
golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=

250
internal/cli/config_input_test.go
View File

@@ -1,233 +1,11 @@
package cli
import (
"errors"
"io"
"os"
"path/filepath"
"strings"
"testing"
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"path/filepath"
"testing"
)
func TestSharedConfigInput(t *testing.T) {
for _, command := range []string{"process", "validate", "print-effective"} {
t.Run(command, func(t *testing.T) {
args := []string{"--config", "main.yml", "--overlay", "testing", "--proposal-profile", "economy", "--modules", "", "--target-sections", "0"}
if command == "process" {
args = append([]string{"transcript.json", "--glossary", "glossary.yml"}, args...)
}
input, help, err := parseConfigInput(command, args, io.Discard)
if err != nil || help {
t.Fatalf("parse: %+v %v %v", input, help, err)
}
if !input.ConfigPathSet || input.ConfigPath != "main.yml" || input.Overlay != "testing" || len(input.Overrides) != 3 {
t.Fatalf("selection: %+v", input)
}
for _, override := range input.Overrides {
switch override.Path.String() {
case ck.Keys("pipeline", "modules").String():
if values, ok := override.Value.([]any); !ok || len(values) != 0 {
t.Fatalf("empty modules: %#v", override.Value)
}
case ck.Keys("chunking", "target_sections").String():
if override.Value != 0 {
t.Fatalf("zero target: %#v", override.Value)
}
}
}
})
}
}
func TestConfigInputRejectsRemovedAndProcessOnlyFlags(t *testing.T) {
for _, command := range []string{"process", "validate", "print-effective"} {
for _, flag := range []string{"--model", "--llm-api-key", "--max-retries", "--llm-concurrency", "--report-json", "--work-dir-retention"} {
args := []string{flag, "value"}
if command == "process" {
args = append([]string{"transcript.json", "--glossary", "glossary.yml"}, args...)
}
if _, _, err := parseConfigInput(command, args, io.Discard); err == nil {
t.Fatalf("%s accepted %s", command, flag)
}
}
}
for _, command := range []string{"validate", "print-effective"} {
if _, _, err := parseConfigInput(command, []string{"--glossary", "glossary.yml"}, io.Discard); err == nil {
t.Fatalf("%s accepted process path", command)
}
if _, _, err := parseConfigInput(command, []string{"transcript.json"}, io.Discard); err == nil {
t.Fatalf("%s accepted positional", command)
}
}
for _, command := range []string{"process", "validate", "print-effective"} {
_, help, err := parseConfigInput(command, []string{"--help"}, io.Discard)
if err != nil || !help {
t.Fatalf("%s help: %v %v", command, help, err)
}
}
}
func TestConfigSelection(t *testing.T) {
dir := t.TempDir()
first, second := filepath.Join(dir, "first.yml"), filepath.Join(dir, "second.yml")
if err := os.WriteFile(second, []byte("version: 2\n"), 0600); err != nil {
t.Fatal(err)
}
path, source, err := selectConfigPath("", false, os.Stat, []string{first, second})
if err != nil || path != second || source != "default" {
t.Fatalf("discovery: %q %q %v", path, source, err)
}
for _, tc := range []struct {
explicit string
set bool
candidates []string
}{
{"", true, []string{second}}, {first, true, []string{second}}, {"", false, []string{first}},
} {
if _, _, err := selectConfigPath(tc.explicit, tc.set, os.Stat, tc.candidates); err == nil {
t.Fatalf("accepted unavailable selection: %+v", tc)
}
}
statDenied := func(string) (os.FileInfo, error) { return nil, os.ErrPermission }
if _, _, err := selectConfigPath("", false, statDenied, []string{first}); !errors.Is(err, os.ErrPermission) {
t.Fatalf("inaccessible candidate: %v", err)
}
path, source, err = selectConfigPath(second, true, os.Stat, []string{first})
if err != nil || path != second || source != "flag" {
t.Fatalf("explicit path: %q %q %v", path, source, err)
}
}
func TestLegacyEnvironmentDoesNotSelectConfig(t *testing.T) {
t.Setenv("AUDITA_CONFIG", filepath.Join(t.TempDir(), "legacy.yml"))
_, _, err := selectConfigPath("", false, os.Stat, []string{filepath.Join(t.TempDir(), "missing.yml")})
if err == nil || !strings.Contains(err.Error(), "no configuration file") {
t.Fatalf("legacy environment changed discovery: %v", err)
}
}
func TestEffectiveViewParsing(t *testing.T) {
for _, view := range []string{"settings", "profiles", "explain"} {
input, _, err := parseConfigInput("print-effective", []string{"--view", view}, io.Discard)
if err != nil || input.View != view {
t.Fatalf("%q: %+v %v", view, input, err)
}
}
if _, _, err := parseConfigInput("print-effective", []string{"--view", "raw"}, io.Discard); err == nil {
t.Fatal("unknown view accepted")
}
if _, _, err := parseConfigInput("validate", []string{"--view", "profiles"}, io.Discard); err == nil {
t.Fatal("validate accepted inspection-only flag")
}
}
func TestSharedLoadingUsesSelectedFileOverlayAndVisitedOverrides(t *testing.T) {
dir := t.TempDir()
if err := os.Mkdir(filepath.Join(dir, "overlays"), 0700); err != nil {
t.Fatal(err)
}
main := filepath.Join(dir, "config.yml")
base := "version: 2\nbackends:\n local:\n endpoint: https://local.invalid/v1\nllm:\n proposal_profile: base\nchunking:\n max_section_tokens: 900\n"
if err := os.WriteFile(main, []byte(base), 0600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "overlays", "testing.yml"), []byte("llm:\n proposal_profile: testing\n"), 0600); err != nil {
t.Fatal(err)
}
for _, command := range []string{"process", "validate", "print-effective"} {
args := []string{"--config", main, "--overlay", "testing", "--min-section-tokens", "500"}
if command == "process" {
args = append([]string{"input.json", "--glossary", "glossary.yml"}, args...)
}
input, _, err := parseConfigInput(command, args, io.Discard)
if err != nil {
t.Fatal(err)
}
selected, err := loadConfigInput(input, command)
if err != nil {
t.Fatal(err)
}
c := selected.Result.Config
if c.LLM.ProposalProfile != "testing" || c.LLM.ValidationProfile != "testing" || c.Chunking.MinSectionTokens != 500 || c.Chunking.MaxSectionTokens != 900 {
t.Fatalf("%s resolved %+v", command, c)
}
}
input, _, err := parseConfigInput("validate", []string{"--overlay", "testing"}, io.Discard)
if err != nil {
t.Fatal(err)
}
if _, err := loadConfigInput(input, "validate"); err == nil {
t.Fatal("validate accepted discovery")
}
input, _, err = parseConfigInput("validate", []string{"--config", main}, io.Discard)
if err != nil {
t.Fatal(err)
}
if _, err := loadConfigInput(input, "validate"); err == nil {
t.Fatal("unmodified invalid lower-precedence section relation accepted")
}
}
func TestReportingAndDebugOverridesRemainIndependent(t *testing.T) {
dir := t.TempDir()
if err := os.Mkdir(filepath.Join(dir, "overlays"), 0700); err != nil {
t.Fatal(err)
}
main := filepath.Join(dir, "config.yml")
base := "version: 2\nbackends:\n local:\n endpoint: https://local.invalid/v1\nllm:\n proposal_profile: base\n"
if err := os.WriteFile(main, []byte(base), 0600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "overlays", "testing.yml"), []byte("reporting:\n enabled: true\n stdout_dir: reports\ndiagnostics:\n debug: true\n"), 0600); err != nil {
t.Fatal(err)
}
defaultInput, _, err := parseConfigInput("validate", []string{"--config", main}, io.Discard)
if err != nil {
t.Fatal(err)
}
defaults, err := loadConfigInput(defaultInput, "validate")
if err != nil {
t.Fatal(err)
}
if !defaults.Result.Config.Reporting.Enabled || defaults.Result.Config.Reporting.StdoutDir != "." || defaults.Result.Config.Diagnostics.Debug {
t.Fatalf("unexpected defaults: %+v", defaults.Result.Config)
}
for _, command := range []string{"process", "validate", "print-effective"} {
args := []string{"--config", main, "--overlay", "testing", "--reporting=false", "--debug=false"}
if command == "process" {
args = append([]string{"input.json", "--glossary", "glossary.yml"}, args...)
}
input, _, err := parseConfigInput(command, args, io.Discard)
if err != nil {
t.Fatal(err)
}
if len(input.Overrides) != 2 || input.Overrides[0].Value != false || input.Overrides[1].Value != false {
t.Fatalf("boolean overrides must remain typed: %+v", input.Overrides)
}
selected, err := loadConfigInput(input, command)
if err != nil {
t.Fatal(err)
}
if selected.Result.Config.Reporting.Enabled || selected.Result.Config.Diagnostics.Debug || selected.Result.Config.Reporting.StdoutDir != "reports" {
t.Fatalf("%s did not honor explicit false: %+v", command, selected.Result.Config)
}
}
processInput, _, err := parseConfigInput("process", []string{"input.json", "--glossary", "glossary.yml", "--config", main, "--overlay", "testing"}, io.Discard)
if err != nil {
t.Fatal(err)
}
processConfig, err := loadConfigInput(processInput, "process")
if err != nil {
t.Fatal(err)
}
if got, err := selectedReportDir(processConfig.Result.Config, filepath.Join("out", "corrected.json"), processInput.ReportDirSet); err != nil || got != "out" {
t.Fatalf("overlay stdout directory redirected file output: %q %v", got, err)
}
}
func TestReportDirectorySelectionOnlyUsesActiveDestination(t *testing.T) {
cfg := config.Config{Reporting: config.Reporting{Enabled: true, StdoutDir: "reports"}}
if got, err := selectedReportDir(cfg, "", false); err != nil || got != "reports" {
@@ -254,27 +32,3 @@ func TestReportDirectorySelectionOnlyUsesActiveDestination(t *testing.T) {
t.Fatalf("disabled reporting probed destination: %q %v", got, err)
}
}
func TestBareBooleanFlagsAndReportDirectoryOverride(t *testing.T) {
input, _, err := parseConfigInput("process", []string{"input.json", "--glossary", "glossary.yml", "--reporting", "--debug", "--report-dir", "reports"}, io.Discard)
if err != nil {
t.Fatal(err)
}
values := map[string]any{}
for _, override := range input.Overrides {
values[override.Path.String()] = override.Value
}
if values[ck.Keys("reporting", "enabled").String()] != true || values[ck.Keys("diagnostics", "debug").String()] != true || values[ck.Keys("reporting", "stdout_dir").String()] != "reports" {
t.Fatalf("incorrect visited overrides: %+v", values)
}
if !input.ReportDirSet {
t.Fatal("explicit --report-dir presence was not recorded")
}
withoutReportDir, _, err := parseConfigInput("process", []string{"input.json", "--glossary", "glossary.yml"}, io.Discard)
if err != nil {
t.Fatal(err)
}
if withoutReportDir.ReportDirSet {
t.Fatal("absent --report-dir was recorded as explicit")
}
}

75
internal/cli/runtime_errors_test.go
View File

@@ -12,65 +12,42 @@ import (
func writeRuntimeErrorConfig(t *testing.T, operatorDocument string) string {
t.Helper()
dir := t.TempDir()
profiles := filepath.Join(dir, "profiles")
if err := os.Mkdir(profiles, 0700); err != nil {
t.Fatal(err)
}
main := filepath.Join(dir, "config.yml")
configText := "version: 2\nprofiles_dir: profiles\nbackends:\n offline:\n endpoint: http://127.0.0.1:1/v1\n credential: AUDITA_TEST_PROVIDER_KEY\nllm:\n proposal_profile: proposal\n"
if err := os.WriteFile(main, []byte(configText), 0600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(profiles, "operator.yml"), []byte(operatorDocument), 0600); err != nil {
t.Fatal(err)
for name, body := range map[string]string{"config.yml": "version: 3\n", "prompts.yml": operatorDocument} {
if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0600); err != nil {
t.Fatal(err)
}
}
return main
}
func TestConfigCommandsClassifyOperatorPromptErrorsAsInputErrors(t *testing.T) {
func TestResourceCommandsClassifyOperatorPromptErrorsAsInputErrors(t *testing.T) {
t.Setenv("AUDITA_TEST_PROVIDER_KEY", "synthetic-cli-redaction-secret")
definitions := map[string]string{
"malformed operator YAML": "profiles:\n proposal: [\n",
"invalid operator template": "profiles:\n proposal:\n backend: offline\n model: local-model\nprompts:\n modules.grammar.proposal:\n v1:\n messages:\n - role: user\n content: '{{'\n",
"missing operator message resource": "profiles:\n proposal:\n backend: offline\n model: local-model\nprompts:\n modules.grammar.proposal:\n v1:\n messages:\n - role: user\n content_file: missing-instruction.md\n",
"malformed operator YAML": "profiles: [\n",
"invalid operator template": "collections:\n audita:\n prompts:\n - id: modules.grammar.proposal\n messages:\n - role: user\n content: '{{'\n output:\n format: text\n validation_mode: none\n",
"missing operator message resource": "collections:\n audita:\n prompts:\n - id: modules.grammar.proposal\n messages:\n - role: user\n content_file: missing-instruction.md\n output:\n format: text\n validation_mode: none\n",
}
commands := []struct {
name string
args []string
}{
{name: "validate", args: []string{"config", "validate"}},
{name: "print effective", args: []string{"config", "print-effective", "--view", "profiles"}},
}
for definition, document := range definitions {
for _, command := range commands {
t.Run(definition+"/"+command.name, func(t *testing.T) {
main := writeRuntimeErrorConfig(t, document)
args := append(append([]string(nil), command.args...), "--config", main)
var stdout, stderr bytes.Buffer
if code := Run(context.Background(), args, &stdout, &stderr); code != 2 {
t.Fatalf("exit=%d, stdout=%q, stderr=%q", code, stdout.String(), stderr.String())
}
if strings.Contains(stderr.String(), "synthetic-cli-redaction-secret") {
t.Fatalf("configured secret leaked: %q", stderr.String())
}
})
}
for name, document := range definitions {
t.Run(name, func(t *testing.T) {
main := writeRuntimeErrorConfig(t, document)
var stdout, stderr bytes.Buffer
if code := Run(context.Background(), []string{"promptkit", "list", "prompts", "--config", main}, &stdout, &stderr); code == 0 {
t.Fatalf("invalid resource accepted: stdout=%q stderr=%q", stdout.String(), stderr.String())
}
if strings.Contains(stderr.String(), "synthetic-cli-redaction-secret") {
t.Fatal("configured secret leaked")
}
})
}
}
func TestConfigCommandsKeepCallerCancellationAsRuntimeExit(t *testing.T) {
t.Setenv("AUDITA_TEST_PROVIDER_KEY", "synthetic-cli-redaction-secret")
main := writeRuntimeErrorConfig(t, "profiles:\n proposal:\n backend: offline\n model: local-model\n")
commands := [][]string{
{"config", "validate", "--config", main},
{"config", "print-effective", "--config", main, "--view", "profiles"},
}
for _, args := range commands {
ctx, cancel := context.WithCancel(context.Background())
cancel()
var stdout, stderr bytes.Buffer
if code := Run(ctx, args, &stdout, &stderr); code != 1 {
t.Fatalf("args=%v exit=%d, stdout=%q, stderr=%q", args, code, stdout.String(), stderr.String())
}
func TestResourceCommandKeepsCallerCancellationAsFailure(t *testing.T) {
main := writeRuntimeErrorConfig(t, "profiles:\n - id: proposal\n model: local-model\n")
ctx, cancel := context.WithCancel(context.Background())
cancel()
var stdout, stderr bytes.Buffer
if code := Run(ctx, []string{"promptkit", "list", "prompts", "--config", main}, &stdout, &stderr); code == 0 {
t.Fatalf("cancellation succeeded: stdout=%q stderr=%q", stdout.String(), stderr.String())
}
}

4
internal/cli/version_test.go
View File

@@ -30,12 +30,10 @@ func TestVersionCommandHelp(t *testing.T) {
args []string
want string
}{
{name: "root help", args: []string{"--help"}, want: "version Print the Audita version"},
{name: "root help", args: []string{"--help"}, want: "version Show application or CommandKit version."},
{name: "help version", args: []string{"help", "version"}, want: "audita version"},
{name: "short flag", args: []string{"version", "-h"}, want: "audita version"},
{name: "long flag", args: []string{"version", "--help"}, want: "audita version"},
{name: "parser help", args: []string{"version", "--h"}, want: "audita version"},
{name: "help short circuits", args: []string{"version", "-h", "--help", "extra"}, want: "audita version"},
} {
t.Run(test.name, func(t *testing.T) {
var stdout, stderr bytes.Buffer

16
internal/core/diagnostics/output_session.go
View File

@@ -51,6 +51,7 @@ type OutputSession struct {
replace func(string, []byte) error
temp *os.File
tempPath string
tempDir string
publishPath string
direct bool
stdout io.Writer
@@ -159,11 +160,16 @@ func (s *OutputSession) Prepare() error {
if err != nil {
return err
}
s.temp, err = os.CreateTemp(filepath.Dir(s.publishPath), ".audita-output-*")
s.tempDir, err = os.MkdirTemp(filepath.Dir(s.publishPath), ".audita-output-*")
if err != nil {
return fmt.Errorf("prepare transcript output %q: %w", s.transcript, err)
}
s.tempPath = s.temp.Name()
s.tempPath = filepath.Join(s.tempDir, "payload")
s.temp, err = os.OpenFile(s.tempPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0644)
if err != nil {
_ = s.Close()
return fmt.Errorf("prepare transcript output %q: %w", s.transcript, err)
}
if info != nil {
if err := s.temp.Chmod(info.Mode().Perm()); err != nil {
_ = s.Close()
@@ -189,6 +195,12 @@ func (s *OutputSession) Close() error {
}
s.tempPath = ""
}
if s.tempDir != "" {
if err := os.Remove(s.tempDir); err != nil && !errors.Is(err, os.ErrNotExist) {
errs = append(errs, err)
}
s.tempDir = ""
}
return errors.Join(errs...)
}

11
internal/framework/contracts/chunk_proposal_composition_test.go
View File

@@ -23,12 +23,7 @@ func (h testChunkProposalHarness) collectEnrichedProposals(
transcript *schema.Transcript,
sections []chunking.Section,
) ([]proposals.EnrichedCorrectionProposal, error) {
runSpecs, err := ResolveModuleRunSpecs([]string{h.module.Key()})
if err != nil {
return nil, err
}
runSpec := runSpecs[0]
runSpec := ModuleRunSpec{ModuleKey: h.module.Key(), InstanceName: h.module.Key()}
runSpec.ReplacementPolicy = h.module.ReplacementPolicy()
out := make([]proposals.EnrichedCorrectionProposal, 0)
@@ -209,7 +204,7 @@ func TestApplyFakeProposalsAcrossMultipleChunksWithStaleSkip(t *testing.T) {
t.Fatalf("collectEnrichedProposals failed: %v", err)
}
result := proposals.ApplyProposals(transcript, enriched, module.ReplacementPolicy())
result := proposals.ApplyProposals(transcript, enriched, module.ReplacementPolicy(), nil)
if len(result.Applied) != 2 {
t.Fatalf("expected 2 applied changes, got %d", len(result.Applied))
@@ -249,7 +244,7 @@ func TestChunkProposalCompositionDoesNotMutateNormalizedTranscript(t *testing.T)
t.Fatalf("collectEnrichedProposals failed: %v", err)
}
_ = proposals.ApplyProposals(transcript, enriched, module.ReplacementPolicy())
_ = proposals.ApplyProposals(transcript, enriched, module.ReplacementPolicy(), nil)
if !reflect.DeepEqual(transcript, before) {
t.Fatalf("normalized transcript mutated\nbefore=%+v\nafter=%+v", before, transcript)

29
internal/framework/contracts/contracts_test.go
View File

@@ -7,6 +7,7 @@ import (
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
)
@@ -80,28 +81,14 @@ func TestInterfaceContractsCompileWithFakes(t *testing.T) {
}
func TestResolveModuleRunSpecs(t *testing.T) {
specs, err := ResolveModuleRunSpecs([]string{"glossary", "homophones", "glossary", "spoken_word", "grammar"})
if err != nil {
t.Fatalf("unexpected error: %v", err)
id := "named"
cfg := &config.Config{Pipeline: config.Pipeline{Stages: []config.Stage{{Module: "glossary", ID: &id}, {Module: "grammar"}, {Module: "glossary"}}}}
specs := ResolveModuleRunSpecs(cfg)
if len(specs) != 3 || specs[0].Index != 0 || specs[0].ID == nil || *specs[0].ID != id || specs[0].InstanceName != "glossary_1" || specs[1].InstanceName != "grammar_2" || specs[2].InstanceName != "glossary_3" {
t.Fatalf("unexpected specs: %+v", specs)
}
expected := []ModuleRunSpec{
{ModuleKey: "glossary", InstanceName: "glossary_1"},
{ModuleKey: "homophones", InstanceName: "homophones"},
{ModuleKey: "glossary", InstanceName: "glossary_2"},
{ModuleKey: "spoken_word", InstanceName: "spoken_word"},
{ModuleKey: "grammar", InstanceName: "grammar"},
}
if !reflect.DeepEqual(specs, expected) {
t.Fatalf("unexpected specs\nexpected: %+v\nactual: %+v", expected, specs)
}
}
func TestResolveModuleRunSpecsRejectsEmptyKey(t *testing.T) {
_, err := ResolveModuleRunSpecs([]string{"glossary", ""})
if err == nil {
t.Fatal("expected error for empty module key")
if len(ResolveModuleRunSpecs(nil)) != 0 {
t.Fatal("nil configuration has run specs")
}
}

3
internal/framework/llm/adapter.go
View File

@@ -100,6 +100,9 @@ func (c *RoleClient) CompleteStructured(parent context.Context, req structuredou
return response, &structuredoutput.Failure{Class: structuredoutput.FailureFatal, Reason: outcomes.ReasonSetupFailure, Message: "unknown structured response schema", Cause: contractErr}
}
vars := prompts.WithSharedVariables(req.Variables)
for name, value := range prompts.SharedVariables() {
vars[name] = value
}
handle, prepErr := c.runtime.Engine.PrepareExecution(ctx, pk.RunRequest{
PromptID: req.PromptID, ProfileID: c.profileID, Vars: vars,
TransientRetries: &c.settings.TransientRetries,

215
internal/framework/llm/adapter_capacity_test.go
View File

@@ -2,216 +2,121 @@ package llm
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
pk "gitea.maximumdirect.net/eric/promptkit"
ck "go.maximumdirect.net/configkit"
)
func TestRolesShareBackendCapacityAndReleaseAfterCancellation(t *testing.T) {
t.Setenv("INHERITED_TEST_KEY", "synthetic-inherited-runtime-secret")
t.Setenv("OPERATOR_BACKEND_KEY", "synthetic-operator-backend-secret")
entered := make(chan struct{}, 3)
release := make(chan struct{})
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if got := r.Header.Get("Authorization"); got != "Bearer synthetic-inherited-runtime-secret" {
t.Errorf("inherited credential header: %q", got)
}
entered <- struct{}{}
<-release
content, _ := json.Marshal(`{"corrections":[]}`)
_, _ = fmt.Fprintf(w, `{"choices":[{"message":{"content":%s}}]}`, content)
}))
defer server.Close()
func capacityFixture(t *testing.T, resources string, transport http.RoundTripper) *Runtime {
t.Helper()
dir := t.TempDir()
if err := os.Mkdir(filepath.Join(dir, "profiles"), 0700); err != nil {
t.Fatal(err)
}
main := filepath.Join(dir, "config.yml")
body := fmt.Sprintf("version: 2\nbackends:\n shared-base:\n endpoint: %s/v1\n credential: INHERITED_TEST_KEY\n concurrency: 1\n queue_capacity: 0\n shared:\n extends: shared-base\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n repair_attempts: 0\n", server.URL)
if err := os.WriteFile(main, []byte(body), 0600); err != nil {
t.Fatal(err)
}
for _, name := range []string{"proposal", "validation"} {
profile := "profiles:\n " + name + ":\n backend: shared\n model: local\n"
if err := os.WriteFile(filepath.Join(dir, "profiles", name+".yaml"), []byte(profile), 0600); err != nil {
for name, content := range map[string]string{"config.yml": "version: 3\n", "prompts.yml": resources} {
if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0600); err != nil {
t.Fatal(err)
}
}
operatorBackend := `backends:
shared:
endpoint: https://operator.invalid/v1
concurrency_limit: 77
queue_capacity: 88
api_key_env: OPERATOR_BACKEND_KEY
`
if err := os.WriteFile(filepath.Join(dir, "profiles", "backend-tuning.yaml"), []byte(operatorBackend), 0600); err != nil {
t.Fatal(err)
}
loaded, err := config.Load(ck.File(main), "")
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
if err != nil {
t.Fatal(err)
}
runtime, err := NewRuntime(context.Background(), loaded.Config, main, server.Client())
runtime, err := NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
if err != nil {
t.Fatal(err)
}
proposal, err := runtime.NewRoleClient("proposal", CallHooks{})
return runtime
}
func capacityResponse(r *http.Request) *http.Response {
return &http.Response{StatusCode: 200, Header: make(http.Header), Request: r, Body: io.NopCloser(strings.NewReader(`{"choices":[{"message":{"content":"{\"corrections\":[]}"}}]}`))}
}
func TestRolesShareBackendCapacityAndReleaseAfterCancellation(t *testing.T) {
entered := make(chan struct{}, 3)
release := make(chan struct{})
transport := transportFunc(func(r *http.Request) (*http.Response, error) {
entered <- struct{}{}
select {
case <-release:
return capacityResponse(r), nil
case <-r.Context().Done():
return nil, r.Context().Err()
}
})
resources := "backends:\n - id: shared\n endpoint: https://example.invalid/v1\n concurrency_limit: 1\n queue_capacity: 1\nprofiles:\n - id: proposal\n backend: shared\n model: local\n - id: validation\n backend: shared\n model: local\n"
runtime := capacityFixture(t, resources, transport)
settings := config.LLM{OperationTimeout: 3 * time.Second}
proposal, err := runtime.NewRoleClient("proposal", "proposal", settings, CallHooks{})
if err != nil {
t.Fatal(err)
}
validation, err := runtime.NewRoleClient("validation", CallHooks{})
validation, err := runtime.NewRoleClient("validation", "validation", settings, CallHooks{})
if err != nil {
t.Fatal(err)
}
request := structuredoutput.Request{PromptID: prompts.PromptIDModuleGlossaryProposal, PromptVersion: prompts.VersionV1, Variables: map[string]string{"TranscriptDescriptionBlock": "", "GlossaryJSON": "[]", "SectionJSON": "[]"}, Schema: responseschema.CorrectionSetKey}
firstDone := make(chan error, 1)
request := adapterRequest(responseschema.CorrectionSetKey)
done := make(chan error, 1)
go func() {
var output map[string]any
_, callErr := proposal.CompleteStructured(context.Background(), request, &output)
firstDone <- callErr
_, err := proposal.CompleteStructured(context.Background(), request, &map[string]any{})
done <- err
}()
select {
case <-entered:
case <-time.After(2 * time.Second):
t.Fatal("first call did not enter provider")
}
var output map[string]any
capacityResponse, err := validation.CompleteStructured(context.Background(), request, &output)
if !errors.Is(err, pk.ErrCapacityExceeded) {
t.Fatalf("other role did not share capacity: %v", err)
}
var capacityFailure *structuredoutput.Failure
if !errors.As(err, &capacityFailure) || capacityFailure.Class != structuredoutput.FailureRecoverable || capacityFailure.Reason != outcomes.ReasonCapacityExceeded || !capacityResponse.Execution.Available || capacityResponse.Execution.ProviderAttempts != 0 {
t.Fatalf("capacity outcome: failure=%#v counts=%+v", capacityFailure, capacityResponse.Execution)
}
select {
case <-entered:
t.Fatal("second role reached provider despite full backend")
default:
}
close(release)
if err := <-firstDone; err != nil {
t.Fatal(err)
}
_, err = validation.CompleteStructured(context.Background(), request, &output)
if err != nil {
t.Fatalf("capacity was not released: %v", err)
}
for len(entered) > 0 {
<-entered
}
release = make(chan struct{})
if err := os.WriteFile(main, []byte(strings.Replace(body, "queue_capacity: 0", "queue_capacity: 1", 1)), 0600); err != nil {
t.Fatal(err)
}
loaded, err = config.Load(ck.File(main), "")
if err != nil {
t.Fatal(err)
}
runtime, err = NewRuntime(context.Background(), loaded.Config, main, server.Client())
if err != nil {
t.Fatal(err)
}
proposal, err = runtime.NewRoleClient("proposal", CallHooks{})
if err != nil {
t.Fatal(err)
}
validation, err = runtime.NewRoleClient("validation", CallHooks{})
if err != nil {
t.Fatal(err)
}
go func() {
var out map[string]any
_, callErr := proposal.CompleteStructured(context.Background(), request, &out)
firstDone <- callErr
}()
select {
case <-entered:
case <-time.After(2 * time.Second):
t.Fatal("first queued-capacity call did not dispatch")
}
queuedCtx, cancelQueued := context.WithTimeout(context.Background(), 30*time.Millisecond)
defer cancelQueued()
_, err = validation.CompleteStructured(queuedCtx, request, &output)
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond)
defer cancel()
_, err = validation.CompleteStructured(ctx, request, &map[string]any{})
if !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("queued call did not honor deadline: %v", err)
}
if !errors.As(err, &capacityFailure) || capacityFailure.Class != structuredoutput.FailureCanceled || capacityFailure.Reason != outcomes.ReasonCanceled {
t.Fatalf("queued parent deadline classification: %#v", capacityFailure)
select {
case <-entered:
t.Fatal("queued call dispatched")
default:
}
close(release)
if err := <-firstDone; err != nil {
if err := <-done; err != nil {
t.Fatal(err)
}
_, err = validation.CompleteStructured(context.Background(), request, &output)
if err != nil {
t.Fatalf("queued cancellation leaked capacity: %v", err)
if _, err := validation.CompleteStructured(context.Background(), request, &map[string]any{}); err != nil {
t.Fatalf("cancellation leaked capacity: %v", err)
}
}
func TestDistinctBackendsHaveIndependentCapacity(t *testing.T) {
entered := make(chan struct{}, 2)
release := make(chan struct{})
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
transport := transportFunc(func(r *http.Request) (*http.Response, error) {
entered <- struct{}{}
<-release
schema := map[string]any{}
_ = json.NewDecoder(r.Body).Decode(&schema)
content, _ := json.Marshal(`{"corrections":[]}`)
_, _ = fmt.Fprintf(w, `{"choices":[{"message":{"content":%s}}]}`, content)
}))
defer server.Close()
dir := t.TempDir()
if err := os.Mkdir(filepath.Join(dir, "profiles"), 0700); err != nil {
t.Fatal(err)
}
main := filepath.Join(dir, "config.yml")
body := fmt.Sprintf("version: 2\nbackends:\n first:\n endpoint: %s/v1\n concurrency: 1\n queue_capacity: 0\n second:\n endpoint: %s/v1\n concurrency: 1\n queue_capacity: 0\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n repair_attempts: 0\n", server.URL, server.URL)
if err := os.WriteFile(main, []byte(body), 0600); err != nil {
t.Fatal(err)
}
for name, backend := range map[string]string{"proposal": "first", "validation": "second"} {
if err := os.WriteFile(filepath.Join(dir, "profiles", name+".yaml"), []byte("profiles:\n "+name+":\n backend: "+backend+"\n model: local\n"), 0600); err != nil {
t.Fatal(err)
}
}
loaded, err := config.Load(ck.File(main), "")
return capacityResponse(r), nil
})
resources := "backends:\n - id: first\n endpoint: https://example.invalid/v1\n concurrency_limit: 1\n queue_capacity: 0\n - id: second\n endpoint: https://example.invalid/v1\n concurrency_limit: 1\n queue_capacity: 0\nprofiles:\n - id: proposal\n backend: first\n model: local\n - id: validation\n backend: second\n model: local\n"
runtime := capacityFixture(t, resources, transport)
settings := config.LLM{OperationTimeout: 3 * time.Second}
first, err := runtime.NewRoleClient("proposal", "proposal", settings, CallHooks{})
if err != nil {
t.Fatal(err)
}
runtime, err := NewRuntime(context.Background(), loaded.Config, main, server.Client())
second, err := runtime.NewRoleClient("validation", "validation", settings, CallHooks{})
if err != nil {
t.Fatal(err)
}
proposal, err := runtime.NewRoleClient("proposal", CallHooks{})
if err != nil {
t.Fatal(err)
}
validation, err := runtime.NewRoleClient("validation", CallHooks{})
if err != nil {
t.Fatal(err)
}
request := structuredoutput.Request{PromptID: prompts.PromptIDModuleGlossaryProposal, PromptVersion: prompts.VersionV1, Variables: map[string]string{"TranscriptDescriptionBlock": "", "GlossaryJSON": "[]", "SectionJSON": "[]"}, Schema: responseschema.CorrectionSetKey}
request := adapterRequest(responseschema.CorrectionSetKey)
done := make(chan error, 2)
go func() {
var out map[string]any
_, e := proposal.CompleteStructured(context.Background(), request, &out)
done <- e
_, err := first.CompleteStructured(context.Background(), request, &map[string]any{})
done <- err
}()
select {
case <-entered:
@@ -219,13 +124,13 @@ func TestDistinctBackendsHaveIndependentCapacity(t *testing.T) {
t.Fatal("first backend did not dispatch")
}
go func() {
var out map[string]any
_, e := validation.CompleteStructured(context.Background(), request, &out)
done <- e
_, err := second.CompleteStructured(context.Background(), request, &map[string]any{})
done <- err
}()
select {
case <-entered:
case <-time.After(2 * time.Second):
close(release)
t.Fatal("second backend was blocked by first")
}
close(release)

267
internal/framework/llm/adapter_test.go
View File

@@ -5,8 +5,8 @@ import (
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
@@ -15,13 +15,13 @@ import (
"testing"
"time"
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
pk "gitea.maximumdirect.net/eric/promptkit"
ck "go.maximumdirect.net/configkit"
pk "go.maximumdirect.net/promptkit"
)
type reply struct {
@@ -30,85 +30,56 @@ type reply struct {
}
func adapterFixture(t *testing.T, replies []reply, retries, repairs int, hook func(*http.Request)) (*Runtime, *atomic.Int32, string) {
return adapterFixtureWithDocuments(t, replies, retries, repairs, hook, "offline", map[string]string{
"definitions.yaml": "profiles:\n base:\n backend: offline\n model: full-model\n",
})
return adapterFixtureWithPrompt(t, replies, retries, repairs, hook, "")
}
func adapterFixtureWithPrompt(t *testing.T, replies []reply, retries, repairs int, hook func(*http.Request), operatorPrompt string) (*Runtime, *atomic.Int32, string) {
documents := map[string]string{
"definitions.yaml": "profiles:\n base:\n backend: offline\n model: full-model\n",
}
if operatorPrompt != "" {
documents["operator-prompt.yaml"] = operatorPrompt
}
return adapterFixtureWithDocuments(t, replies, retries, repairs, hook, "offline", documents)
}
func adapterFixtureWithDocuments(t *testing.T, replies []reply, retries, repairs int, hook func(*http.Request), backendID string, documents map[string]string) (*Runtime, *atomic.Int32, string) {
t.Helper()
var count atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
t.Setenv("ADAPTER_KEY", "adapter-secret")
dir := t.TempDir()
resources := "backends:\n - id: offline\n endpoint: https://provider.invalid/v1\n api_key_env: ADAPTER_KEY\n concurrency_limit: 1\n queue_capacity: 0\nprofiles:\n - id: base\n backend: offline\n model: full-model\n"
for name, body := range map[string]string{"config.yml": "version: 3\n", "prompts.yml": resources + operatorPrompt} {
if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0600); err != nil {
t.Fatal(err)
}
}
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
if err != nil {
t.Fatal(err)
}
transport := transportFunc(func(r *http.Request) (*http.Response, error) {
call := int(count.Add(1))
if hook != nil {
hook(r)
}
step := replies[len(replies)-1]
if call <= len(replies) {
step = replies[call-1]
if err := r.Context().Err(); err != nil {
return nil, err
}
step := replies[min(call-1, len(replies)-1)]
if step.status == 0 {
connection, _, hijackErr := w.(http.Hijacker).Hijack()
if hijackErr != nil {
t.Error(hijackErr)
return
return nil, errors.New("synthetic transport error")
}
body := `{"error":{"message":"provider error"}}`
if step.status == http.StatusOK {
body = `{"choices":[]}`
if step.content != "MALFORMED_ENVELOPE" {
encoded, _ := json.Marshal(step.content)
body = fmt.Sprintf(`{"model":"returned-model","choices":[{"message":{"content":%s}}],"usage":{"prompt_tokens":2,"completion_tokens":3,"total_tokens":5}}`, encoded)
}
_ = connection.Close()
return
}
if step.status != http.StatusOK {
w.WriteHeader(step.status)
_, _ = fmt.Fprint(w, `{"error":{"message":"provider error"}}`)
return
}
if step.content == "MALFORMED_ENVELOPE" {
_, _ = fmt.Fprint(w, `{"choices":[]}`)
return
}
encoded, _ := json.Marshal(step.content)
w.Header().Set("Content-Type", "application/json")
_, _ = fmt.Fprintf(w, `{"model":"returned-model","choices":[{"message":{"content":%s}}],"usage":{"prompt_tokens":2,"completion_tokens":3,"total_tokens":5}}`, encoded)
}))
t.Cleanup(server.Close)
t.Setenv("ADAPTER_KEY", "adapter-secret")
dir := t.TempDir()
profileDirectory := filepath.Join(dir, "profiles")
if err := os.Mkdir(profileDirectory, 0700); err != nil {
t.Fatal(err)
}
base := fmt.Sprintf("version: 2\nprofiles_dir: profiles\nbackends:\n %s:\n endpoint: %s/v1\n credential: ADAPTER_KEY\n concurrency: 1\n queue_capacity: 0\nllm:\n proposal_profile: base\n transient_retries: %d\n repair_attempts: %d\n", backendID, server.URL, retries, repairs)
main := filepath.Join(dir, "config.yml")
if err := os.WriteFile(main, []byte(base), 0600); err != nil {
t.Fatal(err)
}
for name, content := range documents {
if err := os.WriteFile(filepath.Join(profileDirectory, name), []byte(content), 0600); err != nil {
t.Fatal(err)
}
}
loaded, err := config.Load(ck.File(main), "")
if err != nil {
t.Fatal(err)
}
runtime, err := NewRuntime(context.Background(), loaded.Config, main, server.Client())
return &http.Response{StatusCode: step.status, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
})
runtime, err := NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
if err != nil {
t.Fatal(err)
}
runtime.Config.LLM = config.LLM{OperationTimeout: 10 * time.Second, TransientRetries: retries, RepairAttempts: repairs}
return runtime, &count, dir
}
func operatorPromptDocument(system, user string) string {
return fmt.Sprintf("prompts:\n %s:\n v1:\n messages:\n - role: system\n content: %q\n - role: user\n content: %q\n output:\n format: text\n validation_mode: basic\n", prompts.PromptIDModuleGlossaryProposal, system, user)
return fmt.Sprintf("collections:\n audita:\n prompts:\n - id: modules.glossary.proposal\n messages:\n - role: system\n content: %q\n - role: user\n content: %q\n output:\n format: text\n validation_mode: none\n", system, user)
}
func adapterRequest(schema responseschema.Key) structuredoutput.Request {
@@ -118,7 +89,7 @@ func adapterRequest(schema responseschema.Key) structuredoutput.Request {
id = prompts.PromptIDValidatorEditorialReview
vars = map[string]string{"TranscriptDescriptionBlock": "", "PayloadJSON": "[]"}
}
return structuredoutput.Request{PromptID: id, PromptVersion: prompts.VersionV1, Variables: vars, Schema: schema, Context: structuredoutput.DiagnosticContext{ModuleKey: "glossary"}}
return structuredoutput.Request{PromptID: id, Variables: vars, Schema: schema, Context: structuredoutput.DiagnosticContext{ModuleKey: "glossary"}}
}
func TestPreparedAdapterRepairsAndClassifies(t *testing.T) {
@@ -154,7 +125,7 @@ func TestPreparedAdapterRepairsAndClassifies(t *testing.T) {
}
var events []pk.ExecutionEvent
var records []structuredoutput.CallRecord
client, err := runtime.NewRoleClient(role, CallHooks{OnEvent: func(e pk.ExecutionEvent) { events = append(events, e) }, OnCall: func(r structuredoutput.CallRecord) { records = append(records, r) }})
client, err := runtime.NewRoleClient(role, "base", runtime.Config.LLM, CallHooks{OnEvent: func(e pk.ExecutionEvent) { events = append(events, e) }, OnCall: func(r structuredoutput.CallRecord) { records = append(records, r) }})
if err != nil {
t.Fatal(err)
}
@@ -234,7 +205,7 @@ func TestCorrectionSchemaReplacementValuesThroughAdapter(t *testing.T) {
}
payload := `{"corrections":[{` + strings.Join(fields, ",") + `}]}`
runtime, count, _ := adapterFixture(t, []reply{{200, payload}}, 0, tc.repairs, nil)
client, err := runtime.NewRoleClient("proposal", CallHooks{})
client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{})
if err != nil {
t.Fatal(err)
}
@@ -280,7 +251,7 @@ func TestPreparedAdapterMetadataAndTargetGuard(t *testing.T) {
t.Error(err)
}
})
client, err := runtime.NewRoleClient("proposal", CallHooks{})
client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{})
if err != nil {
t.Fatal(err)
}
@@ -314,12 +285,7 @@ func TestPreparedAdapterMetadataAndTargetGuard(t *testing.T) {
if !found {
t.Fatal("hardening policy missing")
}
backend := runtime.Config.Backends["offline"]
backend.Endpoint = "http://127.0.0.1:1/v1"
runtime.Config.Backends["offline"] = backend
if _, err := client.CompleteStructured(context.Background(), req, &out); err == nil || !strings.Contains(err.Error(), "target changed") || count.Load() != 1 {
t.Fatalf("changed application backend target dispatched: %v, %d", err, count.Load())
}
}
func TestOperatorPromptOverridesUsePerCallSchemaAndOptionalSharedText(t *testing.T) {
@@ -365,7 +331,7 @@ func TestOperatorPromptOverridesUsePerCallSchemaAndOptionalSharedText(t *testing
}
posted = append(posted, envelope)
}, operatorPromptDocument(tc.system, tc.user))
client, err := runtime.NewRoleClient("proposal", CallHooks{})
client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{})
if err != nil {
t.Fatal(err)
}
@@ -403,150 +369,9 @@ func TestOperatorPromptOverridesUsePerCallSchemaAndOptionalSharedText(t *testing
}
}
func TestProfilePresenceBackendAuthorityAndProviderParametersReachWire(t *testing.T) {
t.Setenv("CATALOG_TRAP_KEY", "catalog-secret-must-not-be-used")
cases := []struct {
name string
parentTemperature string
childTemperature string
wantTemperature bool
}{
{name: "explicit zero", parentTemperature: " temperature: 0.8\n", childTemperature: " temperature: 0\n", wantTemperature: true},
{name: "omitted", wantTemperature: false},
{name: "null clears inherited control", parentTemperature: " temperature: 0.8\n", childTemperature: " temperature: null\n", wantTemperature: false},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
profileDocument := "profiles:\n \"parent~/partial\":\n backend: openrouter\n model: parent-model\n" + tc.parentTemperature +
" base:\n extends: \"parent~/partial\"\n model: full-model\n timeout_seconds: 0\n" + tc.childTemperature +
" extra_params:\n routing:\n overridden: profile\n profile_only: retained\n cleared: null\n"
operatorBackend := `backends:
openrouter:
endpoint: https://operator.invalid/v1
concurrency_limit: 91
queue_capacity: 92
api_key_env: CATALOG_TRAP_KEY
extra_params:
routing:
inherited: backend
overridden: backend
cleared: remove-me
`
documents := map[string]string{
"unrelated-profile-file.yml": profileDocument,
"provider-tuning.yaml": operatorBackend,
}
var wire map[string]json.RawMessage
runtime, count, _ := adapterFixtureWithDocuments(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, func(r *http.Request) {
if got := r.Header.Get("Authorization"); got != "Bearer adapter-secret" {
t.Errorf("application credential was not authoritative: %q", got)
}
if err := json.NewDecoder(r.Body).Decode(&wire); err != nil {
t.Error(err)
}
}, "openrouter", documents)
profile := runtime.Profiles["proposal"]
if profile.EffectiveModelParams.APIKeyEnv != "" || profile.EffectiveModelParams.Endpoint == "https://operator.invalid/v1" || profile.EffectiveModelParams.TimeoutSeconds != 0 {
t.Fatalf("application backend mapping or zero timeout lost: %+v", profile.EffectiveModelParams)
}
client, err := runtime.NewRoleClient("proposal", CallHooks{})
if err != nil {
t.Fatal(err)
}
var out map[string]any
response, err := client.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &out)
if err != nil {
t.Fatal(err)
}
if count.Load() != 1 {
t.Fatalf("provider calls = %d, want 1", count.Load())
}
temperature, hasTemperature := wire["temperature"]
if hasTemperature != tc.wantTemperature {
t.Fatalf("temperature presence = %t, want %t: %s", hasTemperature, tc.wantTemperature, wire)
}
if tc.wantTemperature {
var value float64
if err := json.Unmarshal(temperature, &value); err != nil || value != 0 {
t.Fatalf("explicit zero temperature = %s (%v)", temperature, err)
}
}
var routing map[string]any
if err := json.Unmarshal(wire["routing"], &routing); err != nil {
t.Fatalf("provider routing fields missing: %s (%v)", wire, err)
}
if routing["inherited"] != "backend" || routing["overridden"] != "profile" || routing["profile_only"] != "retained" {
t.Fatalf("recursively overlaid provider fields = %#v", routing)
}
if value, exists := routing["cleared"]; !exists || value != nil {
t.Fatalf("null did not replace inherited provider field: %#v", routing)
}
if response.Model != "full-model" || response.Prepared == nil {
t.Fatalf("execution did not use selected profile: %+v", response)
}
})
}
}
func TestPreparedAdapterUsesCapturedProfileAndPromptUntilReconstructed(t *testing.T) {
var posted []struct {
Model string `json:"model"`
Messages []structuredoutput.Message `json:"messages"`
}
runtime, count, dir := adapterFixtureWithPrompt(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, func(r *http.Request) {
var envelope struct {
Model string `json:"model"`
Messages []structuredoutput.Message `json:"messages"`
}
if err := json.NewDecoder(r.Body).Decode(&envelope); err != nil {
t.Error(err)
return
}
posted = append(posted, envelope)
}, operatorPromptDocument("Initial operator instructions.", "Initial payload."))
client, err := runtime.NewRoleClient("proposal", CallHooks{})
if err != nil {
t.Fatal(err)
}
profile := "profiles:\n base:\n backend: offline\n model: edited-model\n"
if err := os.WriteFile(filepath.Join(dir, "profiles", "definitions.yaml"), []byte(profile), 0600); err != nil {
t.Fatal(err)
}
updatedPrompt := operatorPromptDocument("Edited operator instructions.", "Edited payload.")
if err := os.WriteFile(filepath.Join(dir, "profiles", "operator-prompt.yaml"), []byte(updatedPrompt), 0600); err != nil {
t.Fatal(err)
}
var out map[string]any
response, err := client.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &out)
if err != nil {
t.Fatal(err)
}
if response.Model != "full-model" || response.Prepared == nil || response.Prepared.Messages[0].Content != "Initial operator instructions." || posted[0].Model != "full-model" {
t.Fatalf("existing runtime did not retain its captured configuration: response=%+v request=%+v", response, posted[0])
}
main := filepath.Join(dir, "config.yml")
newRuntime, err := NewRuntime(context.Background(), runtime.Config, main, http.DefaultClient)
if err != nil {
t.Fatal(err)
}
newClient, err := newRuntime.NewRoleClient("proposal", CallHooks{})
if err != nil {
t.Fatal(err)
}
response, err = newClient.CompleteStructured(context.Background(), adapterRequest(responseschema.CorrectionSetKey), &out)
if err != nil {
t.Fatal(err)
}
if int(count.Load()) != 2 || response.Model != "edited-model" || response.Prepared == nil || response.Prepared.Messages[0].Content != "Edited operator instructions." || posted[1].Model != "edited-model" {
t.Fatalf("new runtime did not capture edited configuration: response=%+v request=%+v", response, posted[1])
}
}
func TestPreparedAdapterRejectsTargetDriftAndPreparationErrors(t *testing.T) {
runtime, count, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, nil)
client, err := runtime.NewRoleClient("proposal", CallHooks{})
client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{})
if err != nil {
t.Fatal(err)
}
@@ -563,7 +388,7 @@ func TestPreparedAdapterCancellationStopsRetryWait(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
var once sync.Once
client, err := runtime.NewRoleClient("proposal", CallHooks{OnEvent: func(e pk.ExecutionEvent) {
client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{OnEvent: func(e pk.ExecutionEvent) {
if e.Kind == pk.ExecutionAttempt && e.Dispatched {
once.Do(cancel)
}
@@ -580,11 +405,9 @@ func TestPreparedAdapterCancellationStopsRetryWait(t *testing.T) {
func TestPreparedAdapterDistinguishesOperationTimeoutFromParentCancellation(t *testing.T) {
blocking := func(*http.Request) { time.Sleep(500 * time.Millisecond) }
runtime, _, _ := adapterFixtureWithDocuments(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, blocking, "offline", map[string]string{
"definitions.yaml": "profiles:\n base:\n backend: offline\n model: full-model\n timeout_seconds: 0\n",
})
runtime, _, _ := adapterFixture(t, []reply{{200, `{"corrections":[]}`}}, 0, 0, blocking)
runtime.Config.LLM.OperationTimeout = 200 * time.Millisecond
client, err := runtime.NewRoleClient("proposal", CallHooks{})
client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{})
if err != nil {
t.Fatal(err)
}
@@ -605,7 +428,7 @@ func TestPreparedAdapterDistinguishesOperationTimeoutFromParentCancellation(t *t
func TestPreparedAdapterClassifiesSetupAndDomainDecodeFailures(t *testing.T) {
runtime, _, _ := adapterFixture(t, []reply{{200, `{"validations":[{"correction_index":9999999999999999999999999,"approved":true,"confidence":0.8,"reason":"ok"}]}`}}, 0, 0, nil)
client, err := runtime.NewRoleClient("validation", CallHooks{})
client, err := runtime.NewRoleClient("validation", "base", runtime.Config.LLM, CallHooks{})
if err != nil {
t.Fatal(err)
}
@@ -638,7 +461,7 @@ func TestPreparedAdapterClassifiesSetupAndDomainDecodeFailures(t *testing.T) {
func TestPreparedAdapterNonRetryableProviderStatus(t *testing.T) {
runtime, count, _ := adapterFixture(t, []reply{{http.StatusBadRequest, ""}}, 2, 0, nil)
client, err := runtime.NewRoleClient("proposal", CallHooks{})
client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, CallHooks{})
if err != nil {
t.Fatal(err)
}

4
internal/framework/llm/observations_test.go
View File

@@ -10,7 +10,7 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/responseschema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
pk "gitea.maximumdirect.net/eric/promptkit"
pk "go.maximumdirect.net/promptkit"
)
func TestObservationSummarySeparatesLogicalCallsAttemptsAndUsage(t *testing.T) {
@@ -93,7 +93,7 @@ func TestObservationCollectorConcurrentAttributionAndSnapshotCopy(t *testing.T)
func TestRoleClientFeedsCollectorAcrossPreparationFailure(t *testing.T) {
runtime, count, _ := adapterFixture(t, []reply{{status: 200, content: `{"corrections":[]}`}}, 0, 0, nil)
collector := NewObservationCollector()
client, err := runtime.NewRoleClient("proposal", collector.Hooks())
client, err := runtime.NewRoleClient("proposal", "base", runtime.Config.LLM, collector.Hooks())
if err != nil {
t.Fatal(err)
}

225
internal/framework/llm/runtime_assembly_test.go
View File

@@ -4,43 +4,17 @@ import (
"context"
"errors"
"io/fs"
"net/http"
"os"
"path"
"path/filepath"
"strings"
"testing"
"testing/fstest"
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
ck "go.maximumdirect.net/configkit"
)
func runtimeAssemblyFixture(t *testing.T, selections string, documents map[string]string) (config.Config, string) {
t.Helper()
dir := t.TempDir()
profileDir := filepath.Join(dir, "profiles")
if err := os.Mkdir(profileDir, 0700); err != nil {
t.Fatal(err)
}
main := filepath.Join(dir, "config.yml")
text := "version: 2\nprofiles_dir: profiles\nbackends:\n offline:\n endpoint: http://127.0.0.1:1/v1\n concurrency: 1\n queue_capacity: 0\nllm:\n" + selections
if err := os.WriteFile(main, []byte(text), 0600); err != nil {
t.Fatal(err)
}
for name, content := range documents {
if err := os.WriteFile(filepath.Join(profileDir, name), []byte(content), 0600); err != nil {
t.Fatal(err)
}
}
loaded, err := config.Load(ck.File(main), "")
if err != nil {
t.Fatal(err)
}
return loaded.Config, main
}
func copyEmbeddedPromptFilesystem(t *testing.T) fstest.MapFS {
t.Helper()
source := prompts.Filesystem()
@@ -62,166 +36,18 @@ func copyEmbeddedPromptFilesystem(t *testing.T) fstest.MapFS {
return files
}
func TestNewRuntimeRequiresCompleteSelectedProfiles(t *testing.T) {
validProposal := " backend: offline\n model: proposal-model\n"
cases := []struct {
name string
selections string
profiles string
role string
field string
}{
{
name: "omitted proposal model with derived validation selection",
selections: " proposal_profile: proposal\n",
profiles: "profiles:\n proposal:\n backend: offline\n",
role: "proposal",
field: "model",
},
{
name: "empty validation model",
selections: " proposal_profile: proposal\n validation_profile: validation\n",
profiles: "profiles:\n proposal:\n" + validProposal + " validation:\n backend: offline\n model: \"\"\n",
role: "validation",
field: "model",
},
{
name: "null proposal model",
selections: " proposal_profile: proposal\n",
profiles: "profiles:\n proposal:\n backend: offline\n model: null\n",
role: "proposal",
field: "model",
},
{
name: "cleared validation endpoint",
selections: " proposal_profile: proposal\n validation_profile: validation\n",
profiles: "profiles:\n proposal:\n" + validProposal + " validation:\n backend: offline\n endpoint: null\n model: validation-model\n",
role: "validation",
field: "endpoint",
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
cfg, main := runtimeAssemblyFixture(t, tc.selections, map[string]string{"definitions.yml": tc.profiles})
_, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}})
if err == nil || !strings.Contains(err.Error(), tc.role) || !strings.Contains(err.Error(), tc.field) {
t.Fatalf("selected incomplete profile was not rejected for %s.%s: %v", tc.role, tc.field, err)
}
})
}
}
func TestNewRuntimeAllowsUnselectedPartialProfiles(t *testing.T) {
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{
"definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n unfinished:\n backend: offline\n",
})
runtime, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}})
if err != nil {
t.Fatal(err)
}
if runtime.Profiles["proposal"].EffectiveModelParams.Model != "proposal-model" || runtime.Profiles["validation"].EffectiveModelParams.Model != "proposal-model" {
t.Fatalf("role profiles were not resolved: %+v", runtime.Profiles)
}
}
func TestNativeProfileInheritancePreservesControlsAndMergesProviderValues(t *testing.T) {
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: \"child profile\"\n", map[string]string{
"unrelated-name.yaml": `profiles:
"parent~/shared":
backend: offline
temperature: 0.4
top_p: 0.7
extra_params:
routing:
retained: from-parent
overridden: parent
cleared: remove-me
"child profile":
extends: "parent~/shared"
model: child-model
extra_params:
routing:
overridden: child
child_only: present
cleared: null
`,
})
runtime, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{}})
if err != nil {
t.Fatal(err)
}
profile := runtime.Profiles["proposal"]
target := profile.EffectiveModelParams
if profile.ProfileID != "child profile" || target.Model != "child-model" || target.Temperature != 0.4 || target.TopP != 0.7 {
t.Fatalf("inherited profile target = %+v", target)
}
routing, ok := target.ExtraParams["routing"].(map[string]any)
if !ok || routing["retained"] != "from-parent" || routing["overridden"] != "child" || routing["child_only"] != "present" {
t.Fatalf("recursively inherited provider values = %#v", target.ExtraParams)
}
if value, exists := routing["cleared"]; !exists || value != nil {
t.Fatalf("null did not replace the inherited provider value: %#v", routing)
}
}
func TestMissingSelectedProfileDirectoryIsOperatorInputError(t *testing.T) {
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{
"definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n",
})
if err := os.RemoveAll(filepath.Join(filepath.Dir(main), cfg.ProfilesDir)); err != nil {
t.Fatal(err)
}
_, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}})
if err == nil || IsAssetError(err) {
t.Fatalf("missing operator profile directory classification: %v", err)
}
}
func TestNewRuntimeRejectsMalformedUnselectedProfilesAndConflicts(t *testing.T) {
cases := []struct {
name string
document map[string]string
selection string
}{
{
name: "malformed unselected definition",
document: map[string]string{"definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n unused:\n temperature: 9\n"},
selection: " proposal_profile: proposal\n",
},
{
name: "undefined selected backend",
document: map[string]string{"definitions.yml": "profiles:\n proposal:\n backend: missing\n model: proposal-model\n"},
selection: " proposal_profile: proposal\n",
},
{
name: "conflicting directory contributions",
document: map[string]string{
"a.yml": "profiles:\n proposal:\n backend: offline\n model: first-model\n",
"b.yml": "profiles:\n proposal:\n model: second-model\n",
},
selection: " proposal_profile: proposal\n",
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
cfg, main := runtimeAssemblyFixture(t, tc.selection, tc.document)
_, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}})
if err == nil || IsAssetError(err) {
t.Fatalf("operator definition classification: %v", err)
}
})
}
}
func TestRuntimeClassifiesEmbeddedAssetsAndHonorsCancellation(t *testing.T) {
_, err := newRuntimeWithPromptFS(context.Background(), config.Config{}, "config.yml", nil, fstest.MapFS{})
loaded, err := config.Load(ck.Input{Source: ck.NoSource()}, "")
if err != nil {
t.Fatal(err)
}
_, err = newRuntimeWithPromptFS(context.Background(), loaded, nil, fstest.MapFS{})
if err == nil || !IsAssetError(err) || !errors.Is(err, fs.ErrNotExist) {
t.Fatalf("embedded discovery error was not preserved as an asset error: %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
_, err = newRuntimeWithPromptFS(ctx, config.Config{}, "config.yml", nil, fstest.MapFS{})
_, err = newRuntimeWithPromptFS(ctx, loaded, nil, fstest.MapFS{})
if !errors.Is(err, context.Canceled) || IsAssetError(err) {
t.Fatalf("caller cancellation classification: %v", err)
}
@@ -229,12 +55,13 @@ func TestRuntimeClassifiesEmbeddedAssetsAndHonorsCancellation(t *testing.T) {
func TestBrokenEmbeddedPromptFailsEmbeddedOnlyConstruction(t *testing.T) {
promptFS := copyEmbeddedPromptFilesystem(t)
assetPath := path.Join(prompts.AssetRoot, prompts.PromptIDModuleGrammarProposal+".yaml")
promptFS[assetPath] = &fstest.MapFile{Data: []byte("prompts:\n modules.grammar.proposal:\n v1:\n messages: []\n output:\n format: json\n validation_mode: json_schema\n schema_path: correction_set.v1.json\n")}
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{
"definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n",
})
_, err := newRuntimeWithPromptFS(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}}, promptFS)
assetPath := path.Join(prompts.AssetRoot, "modules.grammar.proposal.yaml")
promptFS[assetPath] = &fstest.MapFile{Data: []byte("collections:\n audita:\n prompts:\n - id: modules.grammar.proposal\n messages: []\n")}
loaded, err := config.Load(ck.Input{Source: ck.NoSource()}, "")
if err != nil {
t.Fatal(err)
}
_, err = newRuntimeWithPromptFS(context.Background(), loaded, nil, promptFS)
if err == nil || !IsAssetError(err) {
t.Fatalf("broken embedded prompt was not classified as an asset error: %v", err)
}
@@ -242,14 +69,24 @@ func TestBrokenEmbeddedPromptFailsEmbeddedOnlyConstruction(t *testing.T) {
func TestOperatorPromptCanRepairInvalidEmbeddedPrompt(t *testing.T) {
promptFS := copyEmbeddedPromptFilesystem(t)
assetPath := path.Join(prompts.AssetRoot, prompts.PromptIDModuleGrammarProposal+".yaml")
promptFS[assetPath] = &fstest.MapFile{Data: []byte("prompts:\n modules.grammar.proposal:\n v1:\n messages: []\n output:\n format: json\n validation_mode: json_schema\n schema_path: correction_set.v1.json\n")}
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{
"operator.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\nprompts:\n modules.grammar.proposal:\n v1:\n messages:\n - role: system\n content: operator system\n - role: user\n content: operator user\n",
})
runtime, err := newRuntimeWithPromptFS(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}}, promptFS)
assetPath := path.Join(prompts.AssetRoot, "modules.grammar.proposal.yaml")
promptFS[assetPath] = &fstest.MapFile{Data: []byte("collections:\n audita:\n prompts:\n - id: modules.grammar.proposal\n messages: []\n")}
dir := t.TempDir()
for name, content := range map[string]string{
"config.yml": "version: 3\n",
"prompts.yml": "collections:\n audita:\n prompts:\n - id: modules.grammar.proposal\n messages:\n - role: user\n content: operator message\n output:\n format: text\n validation_mode: none\n",
} {
if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0600); err != nil {
t.Fatal(err)
}
}
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
if err != nil {
t.Fatalf("valid final configuration was rejected with an invalid displaced embedded prompt: %v", err)
t.Fatal(err)
}
runtime, err := newRuntimeWithPromptFS(context.Background(), loaded, nil, promptFS)
if err != nil {
t.Fatalf("valid final configuration rejected: %v", err)
}
if runtime.Engine == nil {
t.Fatal("runtime engine was not retained")

258
internal/framework/llm/runtime_inspection_test.go
View File

@@ -9,258 +9,60 @@ import (
"strings"
"testing"
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gopkg.in/yaml.v3"
ck "go.maximumdirect.net/configkit"
)
type rejectingTransport struct{ t *testing.T }
func (r rejectingTransport) RoundTrip(*http.Request) (*http.Response, error) {
r.t.Error("profile inspection contacted a provider")
if r.t != nil {
r.t.Error("resource inspection contacted a provider")
}
return nil, errors.New("provider access forbidden")
}
func profileLayout(t *testing.T, base, overlay string) string {
t.Helper()
dir := t.TempDir()
for _, name := range []string{"profiles", "overlays"} {
if err := os.Mkdir(filepath.Join(dir, name), 0700); err != nil {
t.Fatal(err)
}
}
for name, content := range map[string]string{
"base": "profiles:\n base:\n backend: offline\n model: full-model\n temperature: 0.3\n max_tokens: 500\n top_p: 0.8\n timeout_seconds: 12\n service_tier: priority\n reasoning_effort: medium\n",
"testing": "profiles:\n testing:\n backend: offline\n model: economy-model\n",
} {
if err := os.WriteFile(filepath.Join(dir, "profiles", name+".yaml"), []byte(content), 0600); err != nil {
t.Fatal(err)
}
}
if err := os.WriteFile(filepath.Join(dir, "config.yml"), []byte(base), 0600); err != nil {
t.Fatal(err)
}
if overlay != "" {
if err := os.WriteFile(filepath.Join(dir, "overlays", "testing.yml"), []byte(overlay), 0600); err != nil {
t.Fatal(err)
}
}
return dir
}
const profileBase = "version: 2\nbackends:\n offline:\n endpoint: https://offline.invalid/v1/\n credential: SYNTHETIC_KEY\nllm:\n proposal_profile: base\n"
func TestOfflineProfileAssemblyAndSafeViews(t *testing.T) {
t.Setenv("SYNTHETIC_KEY", "synthetic-value-never-print")
dir := profileLayout(t, profileBase, "llm:\n proposal_profile: testing\n")
client := &http.Client{Transport: rejectingTransport{t}}
for _, tc := range []struct {
name, overlay string
overrides []ck.Override
want string
}{
{"base", "", nil, "full-model"},
{"overlay", "testing", nil, "economy-model"},
{"explicit override", "testing", []ck.Override{{Path: ck.Keys("llm", "proposal_profile"), Value: "base", Flag: "--proposal-profile"}}, "full-model"},
} {
t.Run(tc.name, func(t *testing.T) {
main := filepath.Join(dir, "config.yml")
loaded, err := config.Load(ck.File(main), tc.overlay, tc.overrides...)
if err != nil {
t.Fatal(err)
}
runtime, err := NewRuntime(context.Background(), loaded.Config, main, client)
if err != nil {
t.Fatal(err)
}
if runtime.Engine == nil || runtime.Profiles["proposal"].EffectiveModelParams.Model != tc.want || runtime.Profiles["validation"].EffectiveModelParams.Model != tc.want {
t.Fatalf("roles not resolved: %+v", runtime.Profiles)
}
profiles, err := runtime.SelectedProfilesYAML()
if err != nil {
t.Fatal(err)
}
explain, err := ExplanationYAML(loaded)
if err != nil {
t.Fatal(err)
}
combined := string(profiles) + string(explain)
if !strings.Contains(combined, tc.want) || !strings.Contains(combined, "/llm/validation_profile") {
t.Fatalf("inspection missing selection/provenance: %s", combined)
}
if strings.Contains(combined, "synthetic-value-never-print") || strings.Contains(combined, "https://offline.invalid") {
t.Fatal("safe views exposed credentials or endpoint")
}
if tc.name == "overlay" && !strings.Contains(combined, "overlay") {
t.Fatal("overlay origin missing")
}
if tc.name == "explicit override" && !strings.Contains(combined, "cli") {
t.Fatal("CLI origin missing")
}
if !strings.Contains(string(explain), "validation uses the final proposal profile when omitted") {
t.Fatal("inherited validation derivation missing")
}
if tc.name == "overlay" && !strings.Contains(string(explain), "source: overlays/testing.yml\n layer: overlay") {
t.Fatal("overlay field source missing")
}
if tc.name == "explicit override" && !strings.Contains(string(explain), "source: --proposal-profile\n layer: cli") {
t.Fatal("explicit profile source missing")
}
if tc.name == "base" && (!strings.Contains(string(profiles), "temperature: 0.3") || !strings.Contains(string(profiles), "reasoning_effort: medium")) {
t.Fatal("selected controls missing")
}
})
}
}
func TestSafeViewsOmitProviderAndPromptValues(t *testing.T) {
const providerValue = "private-provider-value"
const promptValue = "private-operator-prompt-text"
runtime, _, dir := adapterFixtureWithDocuments(t, nil, 0, 0, nil, "offline", map[string]string{
"definitions.yaml": "profiles:\n base:\n backend: offline\n model: full-model\n extra_params:\n vendor_extension: " + providerValue + "\n",
"operator-prompt.yaml": operatorPromptDocument(promptValue, promptValue),
})
profiles, err := runtime.SelectedProfilesYAML()
if err != nil {
t.Fatal(err)
}
loaded, err := config.Load(ck.File(filepath.Join(dir, "config.yml")), "")
if err != nil {
t.Fatal(err)
}
explain, err := ExplanationYAML(loaded)
if err != nil {
t.Fatal(err)
}
combined := string(profiles) + string(explain)
for _, value := range []string{providerValue, promptValue, "adapter-secret", "127.0.0.1"} {
if strings.Contains(combined, value) {
t.Fatalf("safe views exposed %q: %s", value, combined)
}
}
}
func TestExplanationYAMLIncludesSafeInheritanceEvidence(t *testing.T) {
t.Setenv("SYNTHETIC_KEY", "synthetic-inherited-secret-never-print")
text := `version: 2
backends:
shared:
endpoint: https://shared.invalid/v1
concurrency: 3
queue_capacity: 0
credential: SYNTHETIC_KEY
offline:
extends: shared
concurrency: 1
llm:
proposal_profile: base
data := `version: 3
glossary_collections:
common:
entries:
- id: a
name: secret-value-not-for-explanation
category: character
derived:
extends: common
`
loaded, err := config.Load(ck.Bytes("config.yml", []byte(text)), "")
if err != nil {
file := filepath.Join(t.TempDir(), "config.yml")
if err := os.WriteFile(file, []byte(data), 0600); err != nil {
t.Fatal(err)
}
loaded, err := config.Load(ck.Input{Source: ck.File(file)}, "")
if err != nil {
t.Fatalf("%v: %+v", err, err.(*ck.LoadError).Diagnostics())
}
explanation, err := ExplanationYAML(loaded)
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(explanation), "synthetic-inherited-secret-never-print") {
t.Fatal("explanation exposed a resolved inherited credential")
text := string(explanation)
if strings.Contains(text, "secret-value-not-for-explanation") {
t.Fatal("explanation exposed an effective value")
}
var records []struct {
Path string `yaml:"path"`
Inheritance []struct {
From string `yaml:"from"`
To string `yaml:"to"`
Reference string `yaml:"reference"`
} `yaml:"inheritance"`
}
if err := yaml.Unmarshal(explanation, &records); err != nil {
t.Fatal(err)
}
var endpoint, credential, concurrency *struct {
Path string `yaml:"path"`
Inheritance []struct {
From string `yaml:"from"`
To string `yaml:"to"`
Reference string `yaml:"reference"`
} `yaml:"inheritance"`
}
for i := range records {
switch records[i].Path {
case "/backends/offline/endpoint":
endpoint = &records[i]
case "/backends/offline/credential":
credential = &records[i]
case "/backends/offline/concurrency":
concurrency = &records[i]
}
}
if endpoint == nil || len(endpoint.Inheritance) != 1 {
t.Fatalf("inherited endpoint evidence missing: %s", explanation)
}
step := endpoint.Inheritance[0]
if step.From != "/backends/shared/endpoint" || step.To != "/backends/offline/endpoint" || step.Reference != "/backends/offline/extends" {
t.Fatalf("inherited endpoint evidence: %+v", step)
}
if credential == nil || len(credential.Inheritance) != 1 {
t.Fatalf("inherited credential evidence missing: %s", explanation)
}
if concurrency == nil || len(concurrency.Inheritance) != 0 {
t.Fatalf("direct child override reported parent inheritance: %+v", concurrency)
if !strings.Contains(text, "/glossary_collections/derived/entries") || !strings.Contains(text, "inheritance:") || !strings.Contains(text, "/glossary_collections/common/entries") {
t.Fatalf("inherited field evidence missing: %s", text)
}
}
func TestProfileAssemblyRejectsUnsafeSelections(t *testing.T) {
t.Setenv("SYNTHETIC_KEY", "synthetic-value-never-print")
dir := profileLayout(t, profileBase, "")
main := filepath.Join(dir, "config.yml")
for _, tc := range []struct{ name, profile string }{
{"endpoint only", "profiles:\n custom:\n endpoint: https://offline.invalid/v1\n model: model\n"},
{"mismatched endpoint", "profiles:\n custom:\n backend: offline\n endpoint: https://elsewhere.invalid/v1\n model: model\n"},
{"profile credential", "profiles:\n custom:\n backend: offline\n model: model\n api_key_env: OTHER_KEY\n"},
} {
t.Run(tc.name, func(t *testing.T) {
if err := os.WriteFile(filepath.Join(dir, "profiles", "custom.yaml"), []byte(tc.profile), 0600); err != nil {
t.Fatal(err)
}
loaded, err := config.Load(ck.File(main), "", ck.Override{Path: ck.Keys("llm", "proposal_profile"), Value: "custom", Flag: "--proposal-profile"})
if err != nil {
t.Fatal(err)
}
if _, err := NewRuntime(context.Background(), loaded.Config, main, &http.Client{Transport: rejectingTransport{t}}); err == nil {
t.Fatal("unsafe profile accepted")
}
})
}
loaded, err := config.Load(ck.File(main), "", ck.Override{Path: ck.Keys("llm", "proposal_profile"), Value: "missing", Flag: "--proposal-profile"})
func TestRuntimeInspectionDoesNotRequireCredentialsOrProvider(t *testing.T) {
loaded, err := config.Load(ck.Input{Source: ck.NoSource()}, "")
if err != nil {
t.Fatal(err)
}
if _, err := NewRuntime(context.Background(), loaded.Config, main, nil); err == nil {
t.Fatal("missing profile accepted")
if _, err := NewRuntime(context.Background(), loaded, &http.Client{Transport: rejectingTransport{t}}); err != nil {
t.Fatal(err)
}
if got := RedactError(errors.New("provider: synthetic-value-never-print"), loaded.Config); strings.Contains(got, "synthetic-value-never-print") {
t.Fatalf("secret leaked: %q", got)
}
}
func TestAbsoluteProfileDirectoryAndMissingSecret(t *testing.T) {
t.Setenv("SYNTHETIC_KEY", "synthetic-value-never-print")
dir := profileLayout(t, profileBase, "")
main := filepath.Join(dir, "config.yml")
absolute := strings.Replace(profileBase, "version: 2\n", "version: 2\nprofiles_dir: "+filepath.Join(dir, "profiles")+"\n", 1)
if err := os.WriteFile(main, []byte(absolute), 0600); err != nil {
t.Fatal(err)
}
loaded, err := config.Load(ck.File(main), "")
if err != nil {
t.Fatal(err)
}
if _, err := NewRuntime(context.Background(), loaded.Config, main, &http.Client{Transport: rejectingTransport{t}}); err != nil {
t.Fatal(err)
}
t.Setenv("SYNTHETIC_KEY", "")
if _, err := config.Load(ck.File(main), ""); err == nil {
t.Fatal("missing surviving secret accepted")
if _, err := ExplanationYAML(nil); err == nil {
t.Fatal("nil configuration accepted")
}
}

34
internal/framework/llm/secrets_test.go
View File

@@ -1,31 +1,45 @@
package llm
import (
"context"
"os"
"path/filepath"
"testing"
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
ck "go.maximumdirect.net/configkit"
)
func TestConfiguredSecretsIncludesUnselectedBackends(t *testing.T) {
if ConfiguredSecrets(nil) != nil {
t.Fatal("nil configuration has secrets")
var empty *Runtime
if empty.ConfiguredSecrets(context.Background()) != nil {
t.Fatal("nil runtime has secrets")
}
t.Setenv("SELECTED_KEY", "selected-secret")
t.Setenv("OTHER_KEY", "unselected-secret")
loaded, err := config.Load(ck.Bytes("config.yml", []byte("version: 2\nbackends:\n selected:\n endpoint: https://selected.invalid/v1\n credential: SELECTED_KEY\n other:\n endpoint: https://other.invalid/v1\n credential: OTHER_KEY\nllm:\n proposal_profile: base\n")), "")
t.Setenv("UNRELATED_KEY", "unrelated-secret")
dir := t.TempDir()
for name, data := range map[string]string{
"config.yml": "version: 3\n",
"prompts.yml": "backends:\n - id: selected\n endpoint: https://selected.invalid/v1\n api_key_env: SELECTED_KEY\n - id: other\n endpoint: https://other.invalid/v1\n api_key_env: OTHER_KEY\nprofiles:\n - id: base\n backend: selected\n model: example\n",
} {
if err := os.WriteFile(filepath.Join(dir, name), []byte(data), 0600); err != nil {
t.Fatal(err)
}
}
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
if err != nil {
t.Fatal(err)
}
got := ConfiguredSecrets(&loaded.Config)
if len(got) != 2 {
t.Fatalf("secrets count = %d", len(got))
runtime, err := NewRuntime(context.Background(), loaded, nil)
if err != nil {
t.Fatal(err)
}
seen := map[string]bool{}
for _, value := range got {
for _, value := range runtime.ConfiguredSecrets(context.Background()) {
seen[value] = true
}
if !seen["selected-secret"] || !seen["unselected-secret"] {
t.Fatal("missing resolved backend secret")
if len(seen) != 2 || !seen["selected-secret"] || !seen["unselected-secret"] {
t.Fatalf("unexpected resolved backend secrets: %v", seen)
}
}

56
internal/framework/llm/source_paths_test.go
View File

@@ -2,58 +2,42 @@ package llm
import (
"context"
"net/http"
"os"
"path/filepath"
"reflect"
"sort"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
ck "go.maximumdirect.net/configkit"
)
func TestRuntimeTracksCapturedConfigurationAndInheritedMessageFiles(t *testing.T) {
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n validation_profile: validation\n", map[string]string{
"selected.yml": "profiles:\n proposal:\n backend: offline\n model: proposal\n validation:\n backend: offline\n model: validation\n",
})
profiles := filepath.Join(filepath.Dir(main), cfg.ProfilesDir)
parentDir := filepath.Join(profiles, "catalog")
childDir := filepath.Join(profiles, "nested", "child")
for _, directory := range []string{parentDir, childDir} {
if err := os.MkdirAll(directory, 0700); err != nil {
t.Fatal(err)
}
}
parent := filepath.Join(parentDir, "arbitrary-name.yaml")
child := filepath.Join(childDir, "override.yaml")
resource := filepath.Join(parentDir, "instruction.md")
parentDocument := "prompts:\n source:\n release/1~draft:\n messages:\n - role: user\n content_file: ./instruction.md\n output:\n format: text\n validation_mode: basic\n"
childDocument := "prompts:\n literal/~child:\n extends: source\n"
for path, content := range map[string]string{
parent: parentDocument,
child: childDocument,
resource: "Captured operator message.\n",
dir := t.TempDir()
for name, content := range map[string]string{
"config.yml": "version: 3\n",
"prompts.yml": "config:\n imports:\n - path: catalog/arbitrary-name.yaml\n required: true\n",
"catalog/arbitrary-name.yaml": "prompts:\n - id: source\n messages:\n - role: user\n content_file: instruction.md\n output:\n format: text\n validation_mode: none\n",
"catalog/instruction.md": "Captured operator message.\n",
"prompts/nested/override.yaml": "prompts:\n - id: literal-child\n extends: {id: source}\n",
} {
if err := os.WriteFile(path, []byte(content), 0600); err != nil {
file := filepath.Join(dir, name)
if err := os.MkdirAll(filepath.Dir(file), 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(file, []byte(content), 0600); err != nil {
t.Fatal(err)
}
}
runtime, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{}})
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
if err != nil {
t.Fatal(err)
}
want := []string{
filepath.Join(profiles, "nested", "child", "override.yaml"),
filepath.Join(profiles, "catalog", "arbitrary-name.yaml"),
filepath.Join(profiles, "selected.yml"),
resource,
}
// Runtime paths are canonical paths from PromptKit's captured sources.
for i := range want {
want[i], err = filepath.EvalSymlinks(want[i])
if err != nil {
t.Fatal(err)
}
runtime, err := NewRuntime(context.Background(), loaded, nil)
if err != nil {
t.Fatal(err)
}
want := []string{filepath.Join(dir, "prompts.yml"), filepath.Join(dir, "catalog/arbitrary-name.yaml"), filepath.Join(dir, "catalog/instruction.md"), filepath.Join(dir, "prompts/nested/override.yaml")}
sort.Strings(want)
if !reflect.DeepEqual(runtime.SourcePaths, want) {
t.Fatalf("captured source paths = %#v, want %#v", runtime.SourcePaths, want)

24
internal/framework/modules/registry_test.go
View File

@@ -6,6 +6,7 @@ import (
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
@@ -41,17 +42,14 @@ func TestUnknownModuleKeyNotRecognized(t *testing.T) {
}
func TestRepeatedRunSpecNamingRemainsDeterministic(t *testing.T) {
specs, err := contracts.ResolveModuleRunSpecs([]string{"glossary", "glossary", "grammar"})
if err != nil {
t.Fatalf("ResolveModuleRunSpecs error: %v", err)
}
if specs[0].InstanceName != "glossary_1" || specs[1].InstanceName != "glossary_2" || specs[2].InstanceName != "grammar" {
specs := contracts.ResolveModuleRunSpecs(&config.Config{Pipeline: config.Pipeline{Stages: []config.Stage{{Module: "glossary"}, {Module: "glossary"}, {Module: "grammar"}}}})
if specs[0].InstanceName != "glossary_1" || specs[1].InstanceName != "glossary_2" || specs[2].InstanceName != "grammar_3" {
t.Fatalf("unexpected instance names: %+v", specs)
}
}
func TestUnsupportedUnknownModuleKeyFailsCleanly(t *testing.T) {
factory := NewFactory(Dependencies{})
factory := NewFactory()
_, err := factory.ModuleForSpec(contracts.ModuleRunSpec{ModuleKey: "unknown", InstanceName: "unknown"})
if err == nil {
t.Fatal("expected unsupported-module error")
@@ -67,7 +65,7 @@ func TestUnsupportedUnknownModuleKeyFailsCleanly(t *testing.T) {
}
func TestRecognizedButMissingConstructorFailsWithInternalRegistryError(t *testing.T) {
factory := NewFactory(Dependencies{})
factory := NewFactory()
// Force a missing constructor state for a known key.
factory.constructors[ModuleKeySpokenWord] = nil
@@ -81,7 +79,7 @@ func TestRecognizedButMissingConstructorFailsWithInternalRegistryError(t *testin
}
func TestGrammarIsRegisteredAndConstructibleByDefault(t *testing.T) {
factory := NewFactory(Dependencies{})
factory := NewFactory()
module, err := factory.ModuleForSpec(contracts.ModuleRunSpec{ModuleKey: ModuleKeyGrammar, InstanceName: ModuleKeyGrammar})
if err != nil {
t.Fatalf("ModuleForSpec error: %v", err)
@@ -92,7 +90,7 @@ func TestGrammarIsRegisteredAndConstructibleByDefault(t *testing.T) {
}
func TestGlossaryIsRegisteredAndConstructibleByDefault(t *testing.T) {
factory := NewFactory(Dependencies{})
factory := NewFactory()
module, err := factory.ModuleForSpec(contracts.ModuleRunSpec{ModuleKey: ModuleKeyGlossary, InstanceName: ModuleKeyGlossary})
if err != nil {
t.Fatalf("ModuleForSpec error: %v", err)
@@ -103,7 +101,7 @@ func TestGlossaryIsRegisteredAndConstructibleByDefault(t *testing.T) {
}
func TestHomophonesIsRegisteredAndConstructibleByDefault(t *testing.T) {
factory := NewFactory(Dependencies{})
factory := NewFactory()
module, err := factory.ModuleForSpec(contracts.ModuleRunSpec{ModuleKey: ModuleKeyHomophones, InstanceName: ModuleKeyHomophones})
if err != nil {
t.Fatalf("ModuleForSpec error: %v", err)
@@ -114,7 +112,7 @@ func TestHomophonesIsRegisteredAndConstructibleByDefault(t *testing.T) {
}
func TestSpokenWordIsRegisteredAndConstructibleByDefault(t *testing.T) {
factory := NewFactory(Dependencies{})
factory := NewFactory()
module, err := factory.ModuleForSpec(contracts.ModuleRunSpec{ModuleKey: ModuleKeySpokenWord, InstanceName: ModuleKeySpokenWord})
if err != nil {
t.Fatalf("ModuleForSpec error: %v", err)
@@ -125,7 +123,7 @@ func TestSpokenWordIsRegisteredAndConstructibleByDefault(t *testing.T) {
}
func TestRegisterConstructorAndConstruct(t *testing.T) {
factory := NewFactory(Dependencies{})
factory := NewFactory()
if err := factory.RegisterConstructor(ModuleKeyGlossary, func(ctx context.Context, req ConstructRequest) (contracts.TranscriptModule, error) {
_ = ctx
if req.RunSpec.InstanceName != "glossary_1" {
@@ -149,7 +147,7 @@ func TestRegisterConstructorAndConstruct(t *testing.T) {
}
func TestRegisterConstructorRejectsUnknownModuleKey(t *testing.T) {
factory := NewFactory(Dependencies{})
factory := NewFactory()
err := factory.RegisterConstructor("unknown", func(ctx context.Context, req ConstructRequest) (contracts.TranscriptModule, error) {
_ = ctx
_ = req

11
internal/framework/processreport/llm_test.go
View File

@@ -7,11 +7,10 @@ import (
"testing"
"time"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/runner"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
pk "gitea.maximumdirect.net/eric/promptkit"
pk "go.maximumdirect.net/promptkit"
)
func TestExecutionReportingGroupsPredispatchFailureAndKeepsSafeMilliseconds(t *testing.T) {
@@ -24,7 +23,7 @@ func TestExecutionReportingGroupsPredispatchFailureAndKeepsSafeMilliseconds(t *t
Events: []llm.ExecutionObservation{{CallID: 1, Kind: string(pk.ExecutionFinished), BackendID: "primary", Attempts: 2, TransientRetries: 1, CapacityWait: 20 * time.Millisecond, GenerationDuration: 500 * time.Millisecond, RetryDelay: 100 * time.Millisecond}},
}
output := &runner.RunOutput{Observations: snapshot, ObservationsAvailable: true, LLMSummary: llm.Summarize(snapshot)}
utilization := BuildUtilization(output, map[string]config.Backend{"primary": {Concurrency: 2, QueueCapacity: 3}, "secondary": {Concurrency: 1}})
utilization := BuildUtilization(output, map[string]*pk.BackendInspection{"primary": {ConcurrencyLimit: intPointer(2), QueueCapacity: intPointer(3)}, "secondary": {ConcurrencyLimit: intPointer(1)}})
if utilization.Backends["primary"].LogicalCalls != 1 || utilization.Backends["primary"].ProviderAttempts != 2 || utilization.Backends["primary"].CapacityWaitMS != 20 || utilization.Backends["primary"].LogicalWallMS != 1500 {
t.Fatalf("primary backend: %+v", utilization.Backends["primary"])
}
@@ -66,7 +65,7 @@ func BenchmarkExecutionReports(b *testing.B) {
})
b.Run(fmt.Sprintf("utilization/%d", n), func(b *testing.B) {
for i := 0; i < b.N; i++ {
BuildUtilization(output, map[string]config.Backend{"local": {Concurrency: 1}})
BuildUtilization(output, map[string]*pk.BackendInspection{"local": {ConcurrencyLimit: intPointer(1)}})
}
})
}
@@ -110,7 +109,7 @@ func TestIndexedReportsPreserveOrderAndPartialUsage(t *testing.T) {
t.Fatalf("call usage: %+v", execution.Calls)
}
output := &runner.RunOutput{Observations: snapshot}
utilization := BuildUtilization(output, map[string]config.Backend{"a": {}, "b": {}})
utilization := BuildUtilization(output, map[string]*pk.BackendInspection{"a": {}, "b": {}})
total := utilization.Backends["a"].Usage.Total
if total.Value != 7 || !total.Known || total.Complete || utilization.Backends["a"].ProviderAttempts != 2 {
t.Fatalf("partial backend usage: %+v", utilization)
@@ -119,3 +118,5 @@ func TestIndexedReportsPreserveOrderAndPartialUsage(t *testing.T) {
t.Fatalf("module grouping: %+v", utilization.Modules)
}
}
func intPointer(value int) *int { return &value }

2
internal/framework/processreport/partial_validation_test.go
View File

@@ -70,7 +70,7 @@ func TestFatalBatchRetainsEarlierDecisionsInLedger(t *testing.T) {
cfg := config.Config{Chunking: config.Chunking{MaxSectionTokens: 8192, MinSectionTokens: 1}, Validation: config.Validation{MaxPromptTokens: 150}}
result, err := runner.New(partialModule{validator}).Run(context.Background(), runner.RunInput{
Config: &cfg, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}}, ValidationLLMClient: client, CaptureCandidates: true,
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}}, CaptureCandidates: true,
})
if !errors.Is(err, origin) || client.calls != 3 {
t.Fatalf("fatal cause/calls: %v %d", err, client.calls)

41
internal/framework/proposal_generation/adapter_integration_test.go
View File

@@ -4,73 +4,74 @@ import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
ck "go.maximumdirect.net/configkit"
)
type proposalTransport func(*http.Request) (*http.Response, error)
func (f proposalTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
func TestRepairedProposalStillFacesDomainChecks(t *testing.T) {
calls := 0
outputs := []string{`{"wrong":true}`, `{"corrections":[{"id":1,"original_text":"absent","corrected_text":"present","confidence":0.4}]}`}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
transport := proposalTransport(func(r *http.Request) (*http.Response, error) {
output := outputs[len(outputs)-1]
if calls < len(outputs) {
output = outputs[calls]
}
calls++
content, _ := json.Marshal(output)
_, _ = fmt.Fprintf(w, `{"choices":[{"message":{"content":%s}}]}`, content)
}))
defer server.Close()
body := fmt.Sprintf(`{"choices":[{"message":{"content":%s}}]}`, content)
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
})
dir := t.TempDir()
if err := os.Mkdir(filepath.Join(dir, "profiles"), 0700); err != nil {
t.Fatal(err)
}
main := filepath.Join(dir, "config.yml")
content := fmt.Sprintf("version: 2\nbackends:\n offline:\n endpoint: %s/v1\nllm:\n proposal_profile: base\n transient_retries: 0\n repair_attempts: 1\n", server.URL)
if err := os.WriteFile(main, []byte(content), 0600); err != nil {
if err := os.WriteFile(main, []byte("version: 3\n"), 0600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "profiles", "base.yaml"), []byte("profiles:\n base:\n backend: offline\n model: local-model\n"), 0600); err != nil {
if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte("profiles:\n - {id: base, endpoint: 'https://example.invalid/v1', model: local-model, api_key_required: false}\n"), 0600); err != nil {
t.Fatal(err)
}
loaded, err := config.Load(ck.File(main), "")
loaded, err := config.Load(ck.Input{Source: ck.File(main)}, "")
if err != nil {
t.Fatal(err)
}
runtime, err := llm.NewRuntime(context.Background(), loaded.Config, main, server.Client())
runtime, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
if err != nil {
t.Fatal(err)
}
client, err := runtime.NewRoleClient("proposal", llm.CallHooks{})
settings := loaded.Config.LLM
settings.TransientRetries, settings.RepairAttempts = 0, 1
client, err := runtime.NewRoleClient("proposal", "base", settings, llm.CallHooks{})
if err != nil {
t.Fatal(err)
}
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "gestures"}}}
vars, err := BuildProposalVariables(transcript, &schema.Glossary{}, 0, "")
vars, err := BuildProposalVariables(transcript, nil, 0, "")
if err != nil {
t.Fatal(err)
}
generated, err := GenerateCandidates(context.Background(), Request{WorkingTranscript: transcript, ModuleKey: "grammar", ModuleInstance: "grammar", ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, Config: &loaded.Config, PromptID: prompts.PromptIDModuleGrammarProposal, PromptVersion: prompts.VersionV1, Variables: vars, LLMClient: client})
generated, err := GenerateCandidates(context.Background(), Request{WorkingTranscript: transcript, ModuleKey: "grammar", ModuleInstance: "grammar", ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, PromptID: prompts.PromptIDModuleGrammarProposal, Variables: vars, LLMClient: client})
if err != nil {
t.Fatal(err)
}
if calls != 2 || len(generated.Enriched) != 1 {
t.Fatalf("repair path: calls=%d, result=%+v", calls, generated)
}
cfg := loaded.Config
cfg.Thresholds.Grammar = .8
input := validators.Request{ModuleKey: "grammar", Config: &cfg, CandidateProposal: generated.Enriched, WorkingTranscript: transcript}
input := validators.Request{ModuleKey: "grammar", Settings: config.Stage{ConfidenceThreshold: .8}, CandidateProposal: generated.Enriched, WorkingTranscript: transcript}
low, err := (validators.ConfidenceThresholdValidator{}).Validate(context.Background(), input)
if err != nil {
t.Fatal(err)

15
internal/framework/proposal_generation/generate_test.go
View File

@@ -10,8 +10,6 @@ import (
"strings"
"testing"
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
@@ -21,6 +19,7 @@ import (
)
type fakeStructuredClient struct {
secrets []string
responses []StructuredCorrectionSet
err error
completion contracts.StructuredCompletionResponse
@@ -51,6 +50,8 @@ func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contr
return response, nil
}
func (f *fakeStructuredClient) ConfiguredSecrets(context.Context) []string { return f.secrets }
type captureDiagnosticsWriter struct {
err error
lastStage string
@@ -69,14 +70,11 @@ func (w *captureDiagnosticsWriter) WriteInteraction(stage string, requestMetadat
func defaultRequest(t *testing.T) Request {
t.Helper()
cfg := config.Config{}
return Request{
ModuleKey: "test_module",
ModuleInstance: "test_module",
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
Config: &cfg,
PromptID: "modules.grammar.proposal",
PromptVersion: "v1",
Variables: map[string]string{"SectionJSON": "[]"},
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 7, Text: "teh"}, {ID: 25, Text: "other"}}},
StartIndex: 0,
@@ -323,13 +321,8 @@ func TestGenerateCandidatesDiagnosticsWrittenAndRedacted(t *testing.T) {
}
t.Setenv("PRIMARY_SECRET", primarySecret)
t.Setenv("VALIDATION_SECRET", validationSecret)
loaded, loadErr := config.Load(ck.Bytes("config.yml", []byte("version: 2\nbackends:\n primary:\n endpoint: https://primary.invalid/v1\n credential: PRIMARY_SECRET\n validation:\n endpoint: https://validation.invalid/v1\n credential: VALIDATION_SECRET\nllm:\n proposal_profile: base\n")), "")
if loadErr != nil {
t.Fatal(loadErr)
}
cfg := loaded.Config
client.secrets = []string{primarySecret, validationSecret}
req := defaultRequest(t)
req.Config = &cfg
req.LLMClient = client
req.DiagnosticsDir = t.TempDir()
req.Variables = map[string]string{"SectionJSON": "include secret " + primarySecret}

24
internal/framework/proposal_generation/module_proposal_test.go
View File

@@ -4,9 +4,11 @@ import (
"context"
"encoding/json"
"errors"
"reflect"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
domain "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
@@ -22,20 +24,20 @@ func TestExecuteModuleProposalBuildsMessagesFromSectionAndDescription(t *testing
},
}
section := contracts.SectionMetadata{Index: 7}
cfg := config.Config{}
cfg := config.Stage{}
cfg.Context.Description = "Hearing transcript with role titles."
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "teh"}}}
glossary := &schema.Glossary{Entries: []schema.GlossaryEntry{{Name: "X"}}}
glossary := []domain.Entry{{ID: "x", Name: "X"}}
var gotSectionIndex int
var gotDescription string
var gotTranscript *schema.Transcript
var gotGlossary *schema.Glossary
var gotGlossary []domain.Entry
out, err := ExecuteModuleProposal(context.Background(), ModuleProposalRequest{
ProposalRequest: contracts.ProposalRequest{
ExecutionContext: contracts.ExecutionContext{
Config: &cfg,
Settings: cfg,
WorkingTranscript: transcript,
Glossary: glossary,
Section: &section,
@@ -48,7 +50,7 @@ func TestExecuteModuleProposalBuildsMessagesFromSectionAndDescription(t *testing
LLMClient: client,
},
PromptID: prompts.PromptIDModuleGrammarProposal,
BuildVariables: func(inTranscript *schema.Transcript, inGlossary *schema.Glossary, sectionIndex int, transcriptDescription string) (map[string]string, error) {
BuildVariables: func(inTranscript *schema.Transcript, inGlossary []domain.Entry, sectionIndex int, transcriptDescription string) (map[string]string, error) {
gotSectionIndex = sectionIndex
gotDescription = transcriptDescription
gotTranscript = inTranscript
@@ -68,8 +70,8 @@ func TestExecuteModuleProposalBuildsMessagesFromSectionAndDescription(t *testing
if gotTranscript != transcript {
t.Fatalf("expected shared transcript pointer")
}
if gotGlossary != glossary {
t.Fatalf("expected shared glossary pointer")
if !reflect.DeepEqual(gotGlossary, glossary) {
t.Fatalf("expected glossary entries")
}
if len(client.calls) != 1 || client.calls[0].PromptID != prompts.PromptIDModuleGrammarProposal || client.calls[0].Context.SectionIndex == nil || *client.calls[0].Context.SectionIndex != 7 {
t.Fatalf("unexpected stage name calls: %+v", client.calls)
@@ -85,7 +87,7 @@ func TestExecuteModuleProposalValidatesInputs(t *testing.T) {
}
_, err := ExecuteModuleProposal(context.Background(), ModuleProposalRequest{
BuildVariables: func(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) (map[string]string, error) {
BuildVariables: func(transcript *schema.Transcript, glossary []domain.Entry, sectionIndex int, transcriptDescription string) (map[string]string, error) {
return nil, nil
},
})
@@ -95,7 +97,7 @@ func TestExecuteModuleProposalValidatesInputs(t *testing.T) {
_, err = ExecuteModuleProposal(context.Background(), ModuleProposalRequest{
PromptID: "missing.prompt.id",
BuildVariables: func(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) (map[string]string, error) {
BuildVariables: func(transcript *schema.Transcript, glossary []domain.Entry, sectionIndex int, transcriptDescription string) (map[string]string, error) {
return map[string]string{"SectionJSON": "[]"}, nil
},
})
@@ -108,7 +110,7 @@ func TestExecuteModuleProposalPropagatesBuilderError(t *testing.T) {
wantErr := errors.New("builder failed")
_, err := ExecuteModuleProposal(context.Background(), ModuleProposalRequest{
PromptID: prompts.PromptIDModuleGlossaryProposal,
BuildVariables: func(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) (map[string]string, error) {
BuildVariables: func(transcript *schema.Transcript, glossary []domain.Entry, sectionIndex int, transcriptDescription string) (map[string]string, error) {
return nil, wantErr
},
})
@@ -128,7 +130,7 @@ func TestExecuteModuleProposalPreservesFatalOperation(t *testing.T) {
LLMClient: client,
},
PromptID: prompts.PromptIDModuleGrammarProposal,
BuildVariables: func(*schema.Transcript, *schema.Glossary, int, string) (map[string]string, error) {
BuildVariables: func(*schema.Transcript, []domain.Entry, int, string) (map[string]string, error) {
return map[string]string{"SectionJSON": "[]"}, nil
},
})

3
internal/framework/proposal_generation/variables_test.go
View File

@@ -4,12 +4,13 @@ import (
"strings"
"testing"
domain "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
)
func TestBuildProposalVariablesPreservesSectionGlossaryAndDescription(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "gestures"}}}
glossary := &schema.Glossary{Entries: []schema.GlossaryEntry{{Name: "Jesters"}}}
glossary := []domain.Entry{{ID: "jesters", Name: "Jesters"}}
variables, err := BuildProposalVariables(transcript, glossary, 0, " Fantasy names ")
if err != nil {
t.Fatal(err)

23
internal/framework/runner/cancellation_test.go
View File

@@ -10,7 +10,6 @@ import (
"testing"
"time"
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
@@ -21,7 +20,8 @@ import (
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/structuredoutput"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
pk "gitea.maximumdirect.net/eric/promptkit"
ck "go.maximumdirect.net/configkit"
pk "go.maximumdirect.net/promptkit"
)
type contextValidator struct {
@@ -47,23 +47,20 @@ func (t *cancelTransport) RoundTrip(req *http.Request) (*http.Response, error) {
func TestValidatorFailureCancelsActiveAndPendingProviderWork(t *testing.T) {
dir := t.TempDir()
if err := os.Mkdir(filepath.Join(dir, "profiles"), 0700); err != nil {
t.Fatal(err)
}
main := filepath.Join(dir, "config.yml")
body := "version: 2\nbackends:\n shared:\n endpoint: http://provider.invalid/v1\n concurrency: 1\n queue_capacity: 1\nllm:\n proposal_profile: base\n transient_retries: 0\n repair_attempts: 0\n"
if err := os.WriteFile(main, []byte(body), 0600); err != nil {
if err := os.WriteFile(main, []byte("version: 3\n"), 0600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "profiles/base.yaml"), []byte("profiles:\n base:\n backend: shared\n model: local\n timeout_seconds: 30\n"), 0600); err != nil {
resources := "backends:\n - id: shared\n endpoint: http://provider.invalid/v1\n concurrency_limit: 1\n queue_capacity: 1\nprofiles:\n - id: base\n backend: shared\n model: local\n"
if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte(resources), 0600); err != nil {
t.Fatal(err)
}
loaded, err := config.Load(ck.File(main), "")
loaded, err := config.Load(ck.Input{Source: ck.File(main)}, "")
if err != nil {
t.Fatal(err)
}
transport := &cancelTransport{entered: make(chan struct{}, 3)}
runtime, err := llm.NewRuntime(context.Background(), loaded.Config, main, &http.Client{Transport: transport})
runtime, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
if err != nil {
t.Fatal(err)
}
@@ -75,7 +72,7 @@ func TestValidatorFailureCancelsActiveAndPendingProviderWork(t *testing.T) {
started <- struct{}{}
}
}
client, err := runtime.NewRoleClient("validation", hooks)
client, err := runtime.NewRoleClient("validation", "base", config.LLM{OperationTimeout: 30 * time.Second}, hooks)
if err != nil {
t.Fatal(err)
}
@@ -101,7 +98,7 @@ func TestValidatorFailureCancelsActiveAndPendingProviderWork(t *testing.T) {
}
var out map[string]any
_, err := client.CompleteStructured(ctx, structuredoutput.Request{
PromptID: prompts.PromptIDValidatorEditorialReview, PromptVersion: prompts.VersionV1,
PromptID: prompts.PromptIDValidatorEditorialReview,
Variables: map[string]string{"TranscriptDescriptionBlock": "", "PayloadJSON": "[]"}, Schema: responseschema.ValidatorDecisionSetKey,
Context: structuredoutput.DiagnosticContext{ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: req.SectionIndex},
}, &out)
@@ -120,7 +117,7 @@ func TestValidatorFailureCancelsActiveAndPendingProviderWork(t *testing.T) {
}
done := make(chan pipelineOutcome, 1)
go func() {
result, err := runModulePipeline(ctx, collectSectionProposalsInput{Module: module, Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar"}, Policy: module.policy, Config: &loaded.Config, Sections: sections})
result, err := runModulePipeline(ctx, collectSectionProposalsInput{Module: module, Spec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar"}, Policy: module.policy, Sections: sections})
done <- pipelineOutcome{result: result, err: err}
}()
select {

92
internal/framework/runner/runner_test.go
View File

@@ -11,8 +11,8 @@ import (
"testing"
"time"
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/modules"
@@ -717,10 +717,7 @@ func TestRunnerDoesNotMutateInputTranscript(t *testing.T) {
}
func TestRunnerRepeatedModuleInstanceNames(t *testing.T) {
specs, err := contracts.ResolveModuleRunSpecs([]string{"glossary", "glossary"})
if err != nil {
t.Fatalf("ResolveModuleRunSpecs error: %v", err)
}
specs := contracts.ResolveModuleRunSpecs(&config.Config{Pipeline: config.Pipeline{Stages: []config.Stage{{Module: "glossary"}, {Module: "glossary"}}}})
if specs[0].InstanceName != "glossary_1" || specs[1].InstanceName != "glossary_2" {
t.Fatalf("unexpected instance names: %+v", specs)
}
@@ -920,10 +917,13 @@ func ptrConfig(c config.Config) *config.Config { return &c }
type fakeStructuredClient struct {
responses []validators.LLMValidationResponse
secrets []string
err error
calls int
}
func (f *fakeStructuredClient) ConfiguredSecrets(context.Context) []string { return f.secrets }
func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
_ = ctx
_ = req
@@ -955,10 +955,9 @@ func TestRunnerLLMValidatorApprovalApplied(t *testing.T) {
}})
cfg := runnerTestConfig()
out, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
ValidationLLMClient: client,
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}},
})
if err != nil {
t.Fatalf("Run error: %v", err)
@@ -978,10 +977,9 @@ func TestRunnerLLMValidatorRejectionPreventsApplication(t *testing.T) {
}})
cfg := runnerTestConfig()
out, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
ValidationLLMClient: client,
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}},
})
if err != nil {
t.Fatalf("Run error: %v", err)
@@ -1007,10 +1005,9 @@ func TestRunnerLLMValidatorMalformedResponseRejectsBatchAndKeepsPartialProgress(
}})
cfg := runnerTestConfig()
out, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {ModuleKey: "m2", InstanceName: "m2"}},
ValidationLLMClient: client,
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m1", InstanceName: "m1"}, {ModuleKey: "m2", InstanceName: "m2", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}},
})
if err != nil {
t.Fatalf("expected malformed validator response to downgrade, got %v", err)
@@ -1058,7 +1055,7 @@ func TestRunnerReconciledSurvivorsCompleteLaterValidation(t *testing.T) {
cfg := runnerTestConfig()
out, err := r.Run(context.Background(), RunInput{
Config: &cfg, Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat bad"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}}, ValidationLLMClient: client,
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}},
})
if err != nil || out.FinalTranscript.Segments[0].Text != "the cat bad" || len(laterIndices) != 2 || laterIndices[0] != 0 || laterIndices[1] != 2 || client.calls != 1 {
t.Fatalf("survivor routing: text=%q later=%v calls=%d err=%v", out.FinalTranscript.Segments[0].Text, laterIndices, client.calls, err)
@@ -1083,10 +1080,9 @@ func TestRunnerLLMValidatorMissingDecisionRejectsBatch(t *testing.T) {
}})
cfg := runnerTestConfig()
_, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
ValidationLLMClient: client,
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}},
})
if err != nil {
t.Fatalf("expected missing decision downgrade, got %v", err)
@@ -1106,10 +1102,9 @@ func TestRunnerLLMValidatorDuplicateDecisionRejectsBatch(t *testing.T) {
}})
cfg := runnerTestConfig()
_, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
ValidationLLMClient: client,
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple."}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}},
})
if err != nil {
t.Fatalf("expected duplicate decision downgrade, got %v", err)
@@ -1139,10 +1134,9 @@ func TestRunnerLLMValidatorBatching(t *testing.T) {
}},
}})
out, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "alpha gamma"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
ValidationLLMClient: client,
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "alpha gamma"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}},
})
if err != nil {
t.Fatalf("unexpected run error: %v", err)
@@ -1161,13 +1155,8 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
validationSecret := "runner-validation-secret"
client := &fakeStructuredClient{responses: []validators.LLMValidationResponse{{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: validationSecret}}}}}
llmValidator, _ := validators.NewLLMBackedValidator("spoken_form_plausibility_review", validators.LLMValidatorTypeSpokenFormPlausibility)
t.Setenv("RUNNER_PRIMARY_KEY", primarySecret)
t.Setenv("RUNNER_VALIDATION_KEY", validationSecret)
loaded, loadErr := config.Load(ck.Bytes("config.yml", []byte("version: 2\nbackends:\n primary:\n endpoint: https://primary.invalid/v1\n credential: RUNNER_PRIMARY_KEY\n validation:\n endpoint: https://validation.invalid/v1\n credential: RUNNER_VALIDATION_KEY\nllm:\n proposal_profile: base\n")), "")
if loadErr != nil {
t.Fatal(loadErr)
}
cfg := loaded.Config
client.secrets = []string{primarySecret, validationSecret}
cfg := runnerTestConfig()
diagDir := t.TempDir()
r := New(fakeFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{key: "m", policy: proposals.ReplacementPolicyRequireUnique, validators: []contracts.Validator{llmValidator}, proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
@@ -1177,8 +1166,7 @@ func TestRunnerLLMValidatorDiagnosticsWrittenAndRedacted(t *testing.T) {
out, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple. " + primarySecret}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
ValidationLLMClient: client,
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ValidationClient: client, Settings: config.Stage{Validation: cfg.Validation}}},
ValidationDiagnosticsDir: diagDir,
})
if err != nil {
@@ -1227,10 +1215,7 @@ func (m proposalGenerationModule) Propose(ctx context.Context, req contracts.Pro
ModuleInstance: req.RunSpec.InstanceName,
ReplacementPolicy: req.RunSpec.ReplacementPolicy,
WorkingTranscript: req.WorkingTranscript,
Config: req.Config,
Glossary: req.Glossary,
PromptID: "modules.grammar.proposal",
PromptVersion: "v1",
Variables: map[string]string{"TranscriptDescriptionBlock": "", "GlossaryJSON": "[]", "SectionJSON": "[]"},
LLMClient: req.LLMClient,
DiagnosticsDir: req.DiagnosticsDir,
@@ -1281,8 +1266,7 @@ func TestRunnerProposalGenerationHelperFlowsThroughPipeline(t *testing.T) {
out, err := r.Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "teh cat"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m"}},
ProposalLLMClient: client,
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "m", InstanceName: "m", ProposalClient: client, Settings: config.Stage{ConfidenceThreshold: cfg.Thresholds.Grammar}}},
ProposalDiagnosticsDir: diagDir,
})
if err != nil {
@@ -1315,13 +1299,11 @@ func TestRunnerGrammarModuleUsesConfidenceThreshold(t *testing.T) {
}
cfg := runnerTestConfig()
cfg.Thresholds.Grammar = 0.9
factory := modules.NewFactory(modules.Dependencies{})
factory := modules.NewFactory()
out, err := New(factory).Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello ,world"}}},
Glossary: &schema.Glossary{},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}},
ProposalLLMClient: client,
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello ,world"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar", ProposalClient: client, Settings: config.Stage{ConfidenceThreshold: cfg.Thresholds.Grammar}}},
})
if err != nil {
t.Fatalf("Run error: %v", err)
@@ -1349,13 +1331,11 @@ func TestRunnerGlossaryModuleUsesConfidenceThreshold(t *testing.T) {
}
cfg := runnerTestConfig()
cfg.Thresholds.Glossary = 0.9
factory := modules.NewFactory(modules.Dependencies{})
factory := modules.NewFactory()
out, err := New(factory).Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures"}}},
Glossary: &schema.Glossary{Entries: []schema.GlossaryEntry{{Name: "Jesters", Category: "faction", Summary: "Faction"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "glossary", InstanceName: "glossary"}},
ProposalLLMClient: client,
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures"}}},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "glossary", InstanceName: "glossary", Glossary: []glossary.Entry{{Name: "Jesters", Category: "faction", Description: "Faction"}}, ProposalClient: client, Settings: config.Stage{ConfidenceThreshold: cfg.Thresholds.Glossary}}},
})
if err != nil {
t.Fatalf("Run error: %v", err)

19
internal/framework/validators/llm_context_test.go
View File

@@ -6,6 +6,7 @@ import (
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/chunking"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
)
@@ -19,11 +20,11 @@ func TestValidationContextCarriesAdjacentEvidenceAndRelevantDefinitions(t *testi
{ID: 4, Text: "Elsewhere."},
}}
req.CandidateProposal[0].TargetSegmentID = 2
req.Glossary = &schema.Glossary{Entries: []schema.GlossaryEntry{
{Name: "Jesters", Summary: "A traveling troupe"},
{Name: "Lyra", Summary: "The host"},
{Name: "Unrelated", Summary: "Not mentioned"},
}}
req.Glossary = []glossary.Entry{
{ID: "jesters", Name: "Jesters", Description: "A traveling troupe"},
{ID: "lyra", Name: "Lyra", Description: "The host"},
{ID: "unrelated", Name: "Unrelated", Description: "Not mentioned"},
}
payload, immediate := BuildLLMValidationRequest("spoken_form_plausibility", LLMValidatorTypeSpokenFormPlausibility, req)
if len(immediate) != 0 || len(payload.Items) != 1 {
t.Fatalf("payload: %+v immediate: %+v", payload, immediate)
@@ -40,10 +41,10 @@ func TestValidationContextCarriesAdjacentEvidenceAndRelevantDefinitions(t *testi
if err := json.Unmarshal([]byte(variables["PayloadJSON"]), &rendered); err != nil {
t.Fatal(err)
}
if strings.Contains(variables["PayloadJSON"], `"id":`) || rendered[0].CorrectionIndex != 1 || payload.Items[0].CorrectionIndex != 7 {
if strings.Contains(variables["PayloadJSON"], `"segment_id":`) || rendered[0].CorrectionIndex != 1 || payload.Items[0].CorrectionIndex != 7 {
t.Fatalf("leaked internal identity or mutated request: %s", variables["PayloadJSON"])
}
if len(rendered[0].NeighboringSegments) != 2 || rendered[0].GlossaryEntries[0].Summary != "A traveling troupe" {
if len(rendered[0].NeighboringSegments) != 2 || rendered[0].GlossaryEntries[0].Description != "A traveling troupe" {
t.Fatalf("rendered context: %+v", rendered)
}
// The context-free endpoint must not reach beyond the current chunk.
@@ -64,7 +65,7 @@ func TestValidationContextFitsBudgetWithoutDiscardingCandidate(t *testing.T) {
}
enriched := base
enriched.NeighboringSegments = []NeighboringSegment{{Position: "following", Text: strings.Repeat("long context ", 300)}}
enriched.GlossaryEntries = []schema.GlossaryEntry{{Name: "cat", Summary: strings.Repeat("long definition ", 300)}}
enriched.GlossaryEntries = []glossary.Entry{{Name: "cat", Description: strings.Repeat("long definition ", 300)}}
fitted := fitValidationContext(enriched, required, estimator)
tokens, err := estimateBatchTokens(estimator, []LLMValidationItem{fitted})
if err != nil || tokens > required || fitted.OriginalSegmentText != base.OriginalSegmentText || fitted.CorrectedSegmentText != base.CorrectedSegmentText {
@@ -80,7 +81,7 @@ func TestValidationContextFitsBudgetWithoutDiscardingCandidate(t *testing.T) {
func TestValidationContextSelectsGlossaryForEachReplacement(t *testing.T) {
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(7, "gestures", "Jesters"), mk(11, "gestures", "Actors")})
req.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "Those gestures are back."}}}
req.Glossary = &schema.Glossary{Entries: []schema.GlossaryEntry{{Name: "Actors"}, {Name: "Jesters"}}}
req.Glossary = []glossary.Entry{{ID: "actors", Name: "Actors"}, {ID: "jesters", Name: "Jesters"}}
payload, immediate := BuildLLMValidationRequest("editorial_review", LLMValidatorTypeEditorialReview, req)
if len(immediate) != 0 || len(payload.Items) != 2 {
t.Fatalf("payload: %+v immediate: %+v", payload, immediate)

41
internal/framework/validators/llm_integration_test.go
View File

@@ -4,19 +4,24 @@ import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
ck "go.maximumdirect.net/configkit"
)
type validationTransport func(*http.Request) (*http.Response, error)
func (f validationTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) {
valid := `{"validations":[{"correction_index":1,"approved":true,"confidence":0.9,"reason":"ok"}]}`
for _, tc := range []struct {
@@ -32,41 +37,38 @@ func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) {
} {
t.Run(tc.name, func(t *testing.T) {
calls := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
transport := validationTransport(func(r *http.Request) (*http.Response, error) {
calls++
output := tc.responses[len(tc.responses)-1]
if calls <= len(tc.responses) {
output = tc.responses[calls-1]
}
if output == "MALFORMED_ENVELOPE" {
_, _ = fmt.Fprint(w, `{"choices":[]}`)
return
body := `{"choices":[]}`
if output != "MALFORMED_ENVELOPE" {
content, _ := json.Marshal(output)
body = fmt.Sprintf(`{"choices":[{"message":{"content":%s}}]}`, content)
}
content, _ := json.Marshal(output)
_, _ = fmt.Fprintf(w, `{"choices":[{"message":{"content":%s}}]}`, content)
}))
defer server.Close()
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
})
dir := t.TempDir()
if err := os.Mkdir(filepath.Join(dir, "profiles"), 0700); err != nil {
t.Fatal(err)
}
main := filepath.Join(dir, "config.yml")
content := fmt.Sprintf("version: 2\nbackends:\n offline:\n endpoint: %s/v1\nllm:\n proposal_profile: base\n transient_retries: 0\n repair_attempts: 1\n", server.URL)
if err := os.WriteFile(main, []byte(content), 0600); err != nil {
if err := os.WriteFile(main, []byte("version: 3\n"), 0600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(dir, "profiles", "base.yaml"), []byte("profiles:\n base:\n backend: offline\n model: local-model\n"), 0600); err != nil {
if err := os.WriteFile(filepath.Join(dir, "prompts.yml"), []byte("profiles:\n - {id: base, endpoint: 'https://example.invalid/v1', model: local-model, api_key_required: false}\n"), 0600); err != nil {
t.Fatal(err)
}
loaded, err := config.Load(ck.File(main), "")
loaded, err := config.Load(ck.Input{Source: ck.File(main)}, "")
if err != nil {
t.Fatal(err)
}
runtime, err := llm.NewRuntime(context.Background(), loaded.Config, main, server.Client())
runtime, err := llm.NewRuntime(context.Background(), loaded, &http.Client{Transport: transport})
if err != nil {
t.Fatal(err)
}
client, err := runtime.NewRoleClient("validation", llm.CallHooks{})
settings := loaded.Config.LLM
settings.TransientRetries, settings.RepairAttempts = 0, 1
client, err := runtime.NewRoleClient("validation", "base", settings, llm.CallHooks{})
if err != nil {
t.Fatal(err)
}
@@ -75,7 +77,6 @@ func TestValidationRepairAndRuntimeFailuresThroughDomainBoundary(t *testing.T) {
t.Fatal(err)
}
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.Config = &loaded.Config
req.LLMClient = client
result, err := validator.Validate(context.Background(), req)
if calls != tc.wantCalls {

6
internal/framework/validators/llm_partial_test.go
View File

@@ -55,7 +55,7 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
validator.SetTokenEstimator(perItemEstimator{})
req := makeReq(candidates)
req.Config.Validation.MaxPromptTokens = 150
req.Settings.Validation.MaxPromptTokens = 150
req.LLMClient = client
result, err := validator.Validate(context.Background(), req)
if err != nil {
@@ -79,7 +79,7 @@ func TestValidationKeepsSuccessfulBatchesAroundUnavailableBatch(t *testing.T) {
nextValidator, _ := NewLLMBackedValidator("meaning_reversal_review", LLMValidatorTypeMeaningReversal)
nextValidator.SetTokenEstimator(perItemEstimator{})
nextReq := makeReq(survivors)
nextReq.Config.Validation.MaxPromptTokens = 150
nextReq.Settings.Validation.MaxPromptTokens = 150
nextReq.LLMClient = nextClient
next, err := nextValidator.Validate(context.Background(), nextReq)
if err != nil || len(next.Decisions) != 2 || len(nextClient.calls) != 2 || *nextClient.calls[0].Context.BatchIndex != 0 || *nextClient.calls[1].Context.BatchIndex != 1 {
@@ -144,7 +144,7 @@ func TestValidationFatalFailureKeepsEarlierFactsAndStops(t *testing.T) {
validator, _ := NewLLMBackedValidator("editorial_review", LLMValidatorTypeEditorialReview)
validator.SetTokenEstimator(perItemEstimator{})
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters"), mk(1, "gestures", "actors"), mk(2, "gestures", "figures")})
req.Config.Validation.MaxPromptTokens = 150
req.Settings.Validation.MaxPromptTokens = 150
req.LLMClient = client
result, err := validator.Validate(context.Background(), req)
if !errors.Is(err, cause) || len(result.Decisions) != 1 || len(result.Operations) != 2 || len(client.calls) != 2 {

13
internal/framework/validators/llm_validators_test.go
View File

@@ -59,15 +59,14 @@ func (f *fakeStructuredLLMClient) CompleteStructured(ctx context.Context, req St
}
func makeReq(candidates []proposals.EnrichedCorrectionProposal) Request {
cfg := config.Config{}
cfg.Validation.MaxPromptTokens = 10000
settings := config.Stage{Validation: config.Validation{MaxPromptTokens: 10000}}
return Request{
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "There were gestures at the temple.", Categories: []string{"narration"}}}},
CandidateProposal: candidates,
ModuleKey: "homophones",
ModuleInstance: "homophones",
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
Config: &cfg,
Settings: settings,
}
}
@@ -314,9 +313,7 @@ func TestLLMBackedValidatorOversizedSingleProposalRejectsOnlyThatProposal(t *tes
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
}
req.LLMClient = client
cfg := config.Config{}
cfg.Validation.MaxPromptTokens = 200
req.Config = &cfg
req.Settings.Validation.MaxPromptTokens = 200
res, err := v.Validate(context.Background(), req)
if err != nil {
@@ -384,7 +381,7 @@ func TestEveryValidationTypeSelectsEmbeddedPromptAndVariables(t *testing.T) {
}
req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")})
req.LLMClient = client
req.Config.Context.Description = " Fantasy scene "
req.Settings.Context.Description = " Fantasy scene "
if _, err := validator.Validate(context.Background(), req); err != nil {
t.Fatal(err)
}
@@ -392,7 +389,7 @@ func TestEveryValidationTypeSelectsEmbeddedPromptAndVariables(t *testing.T) {
t.Fatalf("calls=%d", len(client.calls))
}
call := client.calls[0]
if call.PromptID != tc.prompt || call.PromptVersion != prompts.VersionV1 || call.Schema != responseschema.ValidatorDecisionSetKey || call.Context.Role != "validation" || call.Context.BatchIndex == nil || !strings.Contains(call.Variables["PayloadJSON"], "correction_index") || !strings.Contains(call.Variables["TranscriptDescriptionBlock"], "Fantasy scene") {
if call.PromptID != tc.prompt || call.Schema != responseschema.ValidatorDecisionSetKey || call.Context.Role != "validation" || call.Context.BatchIndex == nil || !strings.Contains(call.Variables["PayloadJSON"], "correction_index") || !strings.Contains(call.Variables["TranscriptDescriptionBlock"], "Fantasy scene") {
t.Fatalf("mapping: %+v", call)
}
})

46
internal/framework/validators/protected_terms_test.go
View File

@@ -2,40 +2,36 @@ package validators
import (
"reflect"
"sort"
"testing"
domain "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
)
func TestExtractProtectedTermsStableDeduplicatedWithPlurals(t *testing.T) {
glossary := &schema.Glossary{
Entries: []schema.GlossaryEntry{
{Name: "Jesters", Aliases: []string{"Jester", " "}, Plural: "Jesters", Category: "faction", Summary: "Faction"},
{Name: "Hrank", Aliases: []string{"hrank", "Hrank"}, Plural: "Hranks", Category: "pc", Summary: "Character"},
{Name: "Lyra", Aliases: []string{}, Plural: "", Category: "npc", Summary: "NPC"},
},
func TestRecognizedTermsDeduplicateAndIncludePlurals(t *testing.T) {
plural := "Hranks"
entries := []domain.Entry{{ID: "jesters", Name: "Jesters", Aliases: []string{"Jester", " "}}, {ID: "hrank", Name: "Hrank", Aliases: []string{"hrank", "Hrank"}, Plural: &plural}, {ID: "lyra", Name: "Lyra"}}
var got []string
for _, term := range domain.Vocabulary(entries, nil) {
got = append(got, term.Spelling)
}
got := ExtractProtectedTerms(glossary)
sort.Strings(got)
want := []string{"Hrank", "Hranks", "Jester", "Jesters", "Jesterss", "Lyra", "Lyras"}
if !reflect.DeepEqual(got, want) {
t.Fatalf("unexpected protected terms\n got: %v\nwant: %v", got, want)
t.Fatalf("got %v want %v", got, want)
}
}
func TestExtractProtectedTermsEmptyGlossary(t *testing.T) {
if terms := ExtractProtectedTerms(&schema.Glossary{}); len(terms) != 0 {
t.Fatalf("expected no terms, got %v", terms)
if len(domain.Vocabulary(nil, nil)) != 0 {
t.Fatal("empty input produced terms")
}
}
func TestValidateProtectedTermsAppliesToNonGlossaryModule(t *testing.T) {
req := Request{
ModuleKey: "grammar",
Glossary: &schema.Glossary{
Entries: []schema.GlossaryEntry{{Name: "Jesters", Category: "faction", Summary: "Faction"}},
},
ModuleKey: "grammar",
Glossary: []domain.Entry{{ID: "jesters", Name: "Jesters", Category: "faction"}},
ProtectedTerms: domain.Vocabulary([]domain.Entry{{ID: "jesters", Name: "Jesters"}}, nil),
CandidateProposal: []proposals.EnrichedCorrectionProposal{
{
CorrectionProposal: proposals.CorrectionProposal{
@@ -52,7 +48,7 @@ func TestValidateProtectedTermsAppliesToNonGlossaryModule(t *testing.T) {
},
},
}
res, err := ValidateProtectedTerms(req, false)
res, err := ValidateProtectedTerms(req)
if err != nil {
t.Fatalf("validator error: %v", err)
}
@@ -64,9 +60,7 @@ func TestValidateProtectedTermsAppliesToNonGlossaryModule(t *testing.T) {
func TestValidateProtectedTermsGlossaryStageAllowsProtectedTermSwap(t *testing.T) {
req := Request{
ModuleKey: "glossary",
Glossary: &schema.Glossary{
Entries: []schema.GlossaryEntry{{Name: "Jesters", Category: "faction", Summary: "Faction"}},
},
Glossary: []domain.Entry{{ID: "jesters", Name: "Jesters", Category: "faction"}},
CandidateProposal: []proposals.EnrichedCorrectionProposal{
{
CorrectionProposal: proposals.CorrectionProposal{
@@ -79,7 +73,7 @@ func TestValidateProtectedTermsGlossaryStageAllowsProtectedTermSwap(t *testing.T
},
},
}
res, err := ValidateProtectedTerms(req, true)
res, err := ValidateProtectedTerms(req)
if err != nil {
t.Fatalf("validator error: %v", err)
}
@@ -103,10 +97,10 @@ func TestProtectedNameRepetitionRequiresSurvivingIdentityAndSpokenWordScope(t *t
t.Run(tc.name, func(t *testing.T) {
req := Request{ModuleKey: tc.module, ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: tc.before}}},
Glossary: &schema.Glossary{Entries: []schema.GlossaryEntry{{Name: "Jesters"}, {Name: "Lyra"}}},
ProtectedTerms: domain.Vocabulary([]domain.Entry{{ID: "jesters", Name: "Jesters"}, {ID: "lyra", Name: "Lyra"}}, nil),
CandidateProposal: []proposals.EnrichedCorrectionProposal{{CorrectionProposal: proposals.CorrectionProposal{TargetSegmentID: 1, OriginalText: tc.original, CorrectedText: tc.replacement, Confidence: 1}}},
}
result, err := ValidateProtectedTerms(req, false)
result, err := ValidateProtectedTerms(req)
if err != nil || len(result.Decisions) != 1 || result.Decisions[0].Approved != tc.allowed {
t.Fatalf("result=%+v err=%v", result, err)
}

14
internal/framework/validators/validators_test.go
View File

@@ -6,6 +6,7 @@ import (
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
domain "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
)
@@ -50,9 +51,7 @@ func TestEnforceDecisionCardinalityUnknownDecision(t *testing.T) {
}
func TestConfidenceThresholdValidator(t *testing.T) {
cfg := config.Config{}
cfg.Thresholds.Grammar = 0.8
req := Request{ModuleKey: "grammar", Config: &cfg, CandidateProposal: []proposals.EnrichedCorrectionProposal{
req := Request{ModuleKey: "grammar", Settings: config.Stage{ConfidenceThreshold: 0.8}, CandidateProposal: []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "teh", "the", 0.9),
mkCandidate(1, 1, "recieve", "receive", 0.7),
}}
@@ -161,16 +160,17 @@ func TestNoEffectValidator(t *testing.T) {
}
func TestValidateProtectedTerms(t *testing.T) {
glossary := &schema.Glossary{Entries: []schema.GlossaryEntry{{Name: "OpenAI", Aliases: []string{"Open AI"}, Plural: "OpenAIs", Category: "brand", Summary: "brand"}}}
entries := []domain.Entry{{ID: "openai", Name: "OpenAI", Aliases: []string{"Open AI"}, Category: "brand", Description: "brand"}}
req := Request{
Glossary: glossary,
ModuleKey: "grammar",
Glossary: entries,
ProtectedTerms: domain.Vocabulary(entries, nil),
ModuleKey: "grammar",
CandidateProposal: []proposals.EnrichedCorrectionProposal{
mkCandidate(0, 1, "OpenAI", "Open A I", 0.9),
mkCandidate(1, 1, "teh", "the", 0.9),
},
}
res, err := ValidateProtectedTerms(req, false)
res, err := ValidateProtectedTerms(req)
if err != nil {
t.Fatalf("Validate error: %v", err)
}

16
internal/modules/glossary/module_test.go
View File

@@ -8,8 +8,7 @@ import (
"strings"
"testing"
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
domain "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
@@ -47,10 +46,9 @@ func tinyTranscript() *schema.Transcript {
}}
}
func tinyGlossary() *schema.Glossary {
return &schema.Glossary{Entries: []schema.GlossaryEntry{
{Name: "Jesters", Aliases: []string{"Jester"}, Category: "faction", Summary: "Guild members", Plural: "Jesters"},
}}
func tinyGlossary() []domain.Entry {
plural := "Jesters"
return []domain.Entry{{Collection: "party", ID: "jesters", Name: "Jesters", Aliases: []string{"Jester"}, Category: "faction", Description: "Guild members", Plural: &plural}}
}
func TestGlossaryModuleReplacementPolicy(t *testing.T) {
@@ -99,11 +97,6 @@ func TestGlossaryModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T)
},
}
t.Setenv("MODULE_SECRET", secret)
loaded, loadErr := config.Load(ck.Bytes("config.yml", []byte("version: 2\nbackends:\n offline:\n endpoint: https://offline.invalid/v1\n credential: MODULE_SECRET\nllm:\n proposal_profile: base\n")), "")
if loadErr != nil {
t.Fatal(loadErr)
}
cfg := loaded.Config
m, err := New()
if err != nil {
t.Fatalf("New error: %v", err)
@@ -111,7 +104,6 @@ func TestGlossaryModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T)
diagDir := t.TempDir()
out, err := m.Propose(context.Background(), contracts.ProposalRequest{
ExecutionContext: contracts.ExecutionContext{
Config: &cfg,
WorkingTranscript: tinyTranscript(),
Glossary: tinyGlossary(),
DiagnosticsDir: diagDir,

16
internal/modules/grammar/module_test.go
View File

@@ -8,8 +8,7 @@ import (
"strings"
"testing"
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
domain "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
@@ -57,10 +56,9 @@ func tinyTranscript() *schema.Transcript {
}}
}
func tinyGlossary() *schema.Glossary {
return &schema.Glossary{Entries: []schema.GlossaryEntry{
{Name: "Jesters", Category: "faction", Summary: "Faction", Aliases: []string{"Jester"}},
}}
func tinyGlossary() []domain.Entry {
plural := "Jesters"
return []domain.Entry{{Collection: "party", ID: "jesters", Name: "Jesters", Aliases: []string{"Jester"}, Category: "faction", Description: "Guild members", Plural: &plural}}
}
func TestGrammarModuleReplacementPolicy(t *testing.T) {
@@ -109,11 +107,6 @@ func TestGrammarModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T) {
},
}
t.Setenv("MODULE_SECRET", secret)
loaded, loadErr := config.Load(ck.Bytes("config.yml", []byte("version: 2\nbackends:\n offline:\n endpoint: https://offline.invalid/v1\n credential: MODULE_SECRET\nllm:\n proposal_profile: base\n")), "")
if loadErr != nil {
t.Fatal(loadErr)
}
cfg := loaded.Config
module, err := New()
if err != nil {
t.Fatalf("New error: %v", err)
@@ -121,7 +114,6 @@ func TestGrammarModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T) {
diagDir := t.TempDir()
out, err := module.Propose(context.Background(), contracts.ProposalRequest{
ExecutionContext: contracts.ExecutionContext{
Config: &cfg,
WorkingTranscript: tinyTranscript(),
Glossary: tinyGlossary(),
DiagnosticsDir: diagDir,

26
internal/modules/homophones/module_test.go
View File

@@ -8,8 +8,8 @@ import (
"strings"
"testing"
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
domain "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
@@ -48,10 +48,9 @@ func tinyTranscript() *schema.Transcript {
}}
}
func tinyGlossary() *schema.Glossary {
return &schema.Glossary{Entries: []schema.GlossaryEntry{
{Name: "Jesters", Aliases: []string{"Jester"}, Category: "faction", Summary: "Guild members", Plural: "Jesters"},
}}
func tinyGlossary() []domain.Entry {
plural := "Jesters"
return []domain.Entry{{Collection: "party", ID: "jesters", Name: "Jesters", Aliases: []string{"Jester"}, Category: "faction", Description: "Guild members", Plural: &plural}}
}
func TestHomophonesModuleReplacementPolicy(t *testing.T) {
@@ -100,11 +99,6 @@ func TestHomophonesModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T
},
}
t.Setenv("MODULE_SECRET", secret)
loaded, loadErr := config.Load(ck.Bytes("config.yml", []byte("version: 2\nbackends:\n offline:\n endpoint: https://offline.invalid/v1\n credential: MODULE_SECRET\nllm:\n proposal_profile: base\n")), "")
if loadErr != nil {
t.Fatal(loadErr)
}
cfg := loaded.Config
m, err := New()
if err != nil {
t.Fatalf("New error: %v", err)
@@ -112,7 +106,6 @@ func TestHomophonesModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T
diagDir := t.TempDir()
out, err := m.Propose(context.Background(), contracts.ProposalRequest{
ExecutionContext: contracts.ExecutionContext{
Config: &cfg,
WorkingTranscript: tinyTranscript(),
Glossary: tinyGlossary(),
DiagnosticsDir: diagDir,
@@ -173,10 +166,8 @@ func TestHomophonesConfidenceThresholdIsUsed(t *testing.T) {
t.Fatal("expected confidence_threshold validator")
}
cfg := config.Config{}
cfg.Thresholds.Homophones = 0.95
result, err := thresholdValidator.Validate(context.Background(), validators.Request{
Config: &cfg,
Settings: config.Stage{ConfidenceThreshold: 0.95},
ModuleKey: "homophones",
CandidateProposal: []proposals.EnrichedCorrectionProposal{
{
@@ -223,12 +214,11 @@ func TestHomophonesProtectedGlossaryTermBehaviorRejectsUnsafeCorrection(t *testi
working := &schema.Transcript{Segments: []schema.Segment{
{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "Jesters are here."},
}}
glossary := &schema.Glossary{Entries: []schema.GlossaryEntry{
{Name: "Jesters", Category: "faction", Summary: "Faction"},
}}
entries := tinyGlossary()
result, err := protectedValidator.Validate(context.Background(), validators.Request{
WorkingTranscript: working,
Glossary: glossary,
Glossary: entries,
ProtectedTerms: domain.Vocabulary(entries, nil),
ModuleKey: "homophones",
CandidateProposal: []proposals.EnrichedCorrectionProposal{
{

26
internal/modules/spoken_word/module_test.go
View File

@@ -8,8 +8,8 @@ import (
"strings"
"testing"
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
domain "gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposal_generation"
@@ -48,10 +48,9 @@ func tinyTranscript() *schema.Transcript {
}}
}
func tinyGlossary() *schema.Glossary {
return &schema.Glossary{Entries: []schema.GlossaryEntry{
{Name: "Jesters", Aliases: []string{"Jester"}, Category: "faction", Summary: "Guild members", Plural: "Jesters"},
}}
func tinyGlossary() []domain.Entry {
plural := "Jesters"
return []domain.Entry{{Collection: "party", ID: "jesters", Name: "Jesters", Aliases: []string{"Jester"}, Category: "faction", Description: "Guild members", Plural: &plural}}
}
func TestSpokenWordModuleReplacementPolicy(t *testing.T) {
@@ -100,11 +99,6 @@ func TestSpokenWordModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T
},
}
t.Setenv("MODULE_SECRET", secret)
loaded, loadErr := config.Load(ck.Bytes("config.yml", []byte("version: 2\nbackends:\n offline:\n endpoint: https://offline.invalid/v1\n credential: MODULE_SECRET\nllm:\n proposal_profile: base\n")), "")
if loadErr != nil {
t.Fatal(loadErr)
}
cfg := loaded.Config
m, err := New()
if err != nil {
t.Fatalf("New error: %v", err)
@@ -112,7 +106,6 @@ func TestSpokenWordModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T
diagDir := t.TempDir()
out, err := m.Propose(context.Background(), contracts.ProposalRequest{
ExecutionContext: contracts.ExecutionContext{
Config: &cfg,
WorkingTranscript: tinyTranscript(),
Glossary: tinyGlossary(),
DiagnosticsDir: diagDir,
@@ -173,10 +166,8 @@ func TestSpokenWordConfidenceThresholdIsUsed(t *testing.T) {
t.Fatal("expected confidence_threshold validator")
}
cfg := config.Config{}
cfg.Thresholds.SpokenWord = 0.95
result, err := thresholdValidator.Validate(context.Background(), validators.Request{
Config: &cfg,
Settings: config.Stage{ConfidenceThreshold: 0.95},
ModuleKey: "spoken_word",
CandidateProposal: []proposals.EnrichedCorrectionProposal{
{
@@ -223,12 +214,11 @@ func TestSpokenWordProtectedGlossaryTermBehaviorRejectsUnsafeCorrection(t *testi
working := &schema.Transcript{Segments: []schema.Segment{
{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "Jesters are here."},
}}
glossary := &schema.Glossary{Entries: []schema.GlossaryEntry{
{Name: "Jesters", Category: "faction", Summary: "Faction"},
}}
entries := tinyGlossary()
result, err := protectedValidator.Validate(context.Background(), validators.Request{
WorkingTranscript: working,
Glossary: glossary,
Glossary: entries,
ProtectedTerms: domain.Vocabulary(entries, nil),
ModuleKey: "spoken_word",
CandidateProposal: []proposals.EnrichedCorrectionProposal{
{

33
internal/validators/protected_terms/validator_test.go
View File

@@ -4,7 +4,7 @@ import (
"context"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
frameworkvalidators "gitea.maximumdirect.net/eric/audita/v3/internal/framework/validators"
@@ -27,12 +27,8 @@ func proposal(index int, moduleKey, original, corrected string) proposals.Enrich
}
}
func tinyGlossary() *schema.Glossary {
return &schema.Glossary{
Entries: []schema.GlossaryEntry{
{Name: "Jesters", Category: "faction", Summary: "Faction"},
},
}
func tinyGlossary() []glossary.Entry {
return []glossary.Entry{{Collection: "party", ID: "jesters", Name: "Jesters", Category: "faction", Description: "Faction"}}
}
func TestNewReportsStableKeyAndDeterministicClass(t *testing.T) {
@@ -83,8 +79,9 @@ func TestNewRejectsProtectedTermChangeOutsideGlossaryStage(t *testing.T) {
}
req := contracts.ValidationRequest{
ModuleKey: "homophones",
Glossary: tinyGlossary(),
ModuleKey: "homophones",
Glossary: tinyGlossary(),
ProtectedTerms: glossary.Vocabulary(tinyGlossary(), nil),
CandidateProposal: []proposals.EnrichedCorrectionProposal{
proposal(0, "homophones", "Jesters", "Gestures"),
},
@@ -105,9 +102,9 @@ func TestNewRejectsProtectedTermChangeOutsideGlossaryStage(t *testing.T) {
}
func TestNewGlossaryStageRejectsProtectedTermChange(t *testing.T) {
v, err := NewGlossaryStage()
v, err := New()
if err != nil {
t.Fatalf("NewGlossaryStage: %v", err)
t.Fatalf("New: %v", err)
}
if v.Name() != Key {
t.Fatalf("expected key %q, got %q", Key, v.Name())
@@ -117,8 +114,9 @@ func TestNewGlossaryStageRejectsProtectedTermChange(t *testing.T) {
}
req := contracts.ValidationRequest{
ModuleKey: "glossary",
Glossary: tinyGlossary(),
ModuleKey: "glossary",
Glossary: tinyGlossary(),
ProtectedTerms: glossary.Vocabulary(tinyGlossary(), nil),
CandidateProposal: []proposals.EnrichedCorrectionProposal{
proposal(0, "glossary", "Jesters", "Gestures"),
},
@@ -139,14 +137,15 @@ func TestNewGlossaryStageRejectsProtectedTermChange(t *testing.T) {
}
func TestNewGlossaryStageAllowsProtectedTermInsertion(t *testing.T) {
v, err := NewGlossaryStage()
v, err := New()
if err != nil {
t.Fatalf("NewGlossaryStage: %v", err)
t.Fatalf("New: %v", err)
}
req := contracts.ValidationRequest{
ModuleKey: "glossary",
Glossary: tinyGlossary(),
ModuleKey: "glossary",
Glossary: tinyGlossary(),
ProtectedTerms: glossary.Vocabulary(tinyGlossary(), nil),
CandidateProposal: []proposals.EnrichedCorrectionProposal{
proposal(0, "glossary", "gestures", "Jesters"),
},

13
internal/validators/registry_test.go
View File

@@ -4,8 +4,8 @@ import (
"context"
"testing"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/glossary"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
"gitea.maximumdirect.net/eric/audita/v3/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/v3/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/v3/internal/validators/confidence_threshold"
@@ -121,9 +121,7 @@ func TestRegistryProtectedTermsUsesNonGlossaryStageBehavior(t *testing.T) {
req := contracts.ValidationRequest{
ModuleKey: "glossary",
Glossary: &schema.Glossary{
Entries: []schema.GlossaryEntry{{Name: "Jesters", Category: "faction", Summary: "Protected"}},
},
Glossary: []glossary.Entry{{Collection: "party", ID: "jesters", Name: "Jesters", Category: "faction", Description: "Protected"}},
CandidateProposal: []proposals.EnrichedCorrectionProposal{
{
CorrectionProposal: proposals.CorrectionProposal{
@@ -169,10 +167,9 @@ func TestResolveBuiltInChainUsesGlossaryStageProtectedTermsBehavior(t *testing.T
}
req := contracts.ValidationRequest{
ModuleKey: "glossary",
Glossary: &schema.Glossary{
Entries: []schema.GlossaryEntry{{Name: "Jesters", Category: "faction", Summary: "Protected"}},
},
ModuleKey: "glossary",
ProtectedTerms: glossary.Vocabulary([]glossary.Entry{{Collection: "party", ID: "jesters", Name: "Jesters"}}, nil),
Glossary: []glossary.Entry{{Collection: "party", ID: "jesters", Name: "Jesters", Category: "faction", Description: "Protected"}},
CandidateProposal: []proposals.EnrichedCorrectionProposal{
{
CorrectionProposal: proposals.CorrectionProposal{