audita/examples/library-adoption/config_test.go

package main

import (
	"bytes"
	"context"
	"encoding/json"
	"os"
	"path/filepath"
	"strings"
	"testing"

	ck "gitea.maximumdirect.net/eric/ConfigKit"
)

func TestConfigurationContract(t *testing.T) {
	original, err := os.ReadFile("config.yaml")
	if err != nil {
		t.Fatal(err)
	}
	for _, tc := range []struct {
		name string
		edit func(string) string
		key  string
		bad  bool
	}{
		{"complete", func(s string) string { return s }, "synthetic-test-secret", false},
		{"empty credential", func(s string) string { return s }, "", true},
		{"quoted version", func(s string) string { return strings.Replace(s, "version: 2", "version: '2'", 1) }, "key", true},
		{"version one", func(s string) string { return strings.Replace(s, "version: 2", "version: 1", 1) }, "key", true},
		{"missing version", func(s string) string { return strings.Replace(s, "version: 2\n", "", 1) }, "key", true},
		{"unknown field", func(s string) string { return s + "unknown: true\n" }, "key", true},
		{"omitted validation profile", func(s string) string { return strings.Replace(s, "  validation_profile: assessment\n", "", 1) }, "key", false},
		{"empty validation profile", func(s string) string {
			return strings.Replace(s, "validation_profile: assessment", "validation_profile: ''", 1)
		}, "key", true},
		{"null validation profile", func(s string) string {
			return strings.Replace(s, "validation_profile: assessment", "validation_profile: null", 1)
		}, "key", true},
		{"omitted backend limits", func(s string) string {
			return strings.Replace(strings.Replace(s, "    concurrency: 4\n", "", 1), "    queue_capacity: 8\n", "", 1)
		}, "key", false},
		{"zero backend limit", func(s string) string { return strings.Replace(s, "concurrency: 4", "concurrency: 0", 1) }, "key", true},
		{"zero waiting capacity", func(s string) string { return strings.Replace(s, "queue_capacity: 8", "queue_capacity: 0", 1) }, "key", false},
		{"null optional credential", func(s string) string {
			return strings.Replace(s, "credential: AUDITA_PROTOTYPE_KEY", "credential: null", 1)
		}, "", false},
	} {
		t.Run(tc.name, func(t *testing.T) {
			t.Setenv("AUDITA_PROTOTYPE_KEY", tc.key)
			loaded, err := loadConfig(ck.Bytes("example", []byte(tc.edit(string(original)))), "")
			if tc.bad {
				if err == nil {
					t.Fatal("expected rejection")
				}
				return
			}
			if err != nil {
				t.Fatal(err)
			}
			if loaded.Config.Version != 2 {
				t.Fatal("version not decoded")
			}
			if tc.name == "omitted validation profile" && loaded.Config.LLM.ValidationProfile != loaded.Config.LLM.ProposalProfile {
				t.Fatal("inheritance failed")
			}
			if tc.name == "omitted backend limits" && (loaded.Config.Backends["offline"].Concurrency != 1 || loaded.Config.Backends["offline"].QueueCapacity != 1024) {
				t.Fatal("finite defaults missing")
			}
			var effective bytes.Buffer
			if err := loaded.WriteEffective(&effective); err != nil {
				t.Fatal(err)
			}
			if strings.Contains(effective.String(), "synthetic-test-secret") {
				t.Fatal("credential exposed")
			}
			r, err := newRuntime(loaded.Config, ".", demonstrationClient())
			if err != nil {
				t.Fatal(err)
			}
			p, err := r.Engine.InspectProfile(context.Background(), "proposal")
			if err != nil {
				t.Fatal(err)
			}
			raw, _ := json.Marshal(p)
			if strings.Contains(string(raw), "synthetic-test-secret") {
				t.Fatal("profile exposed credential")
			}
		})
	}
	t.Setenv("AUDITA_PROTOTYPE_KEY", "")
	if err := os.Unsetenv("AUDITA_PROTOTYPE_KEY"); err != nil {
		t.Fatal(err)
	}
	if _, err := loadConfig(ck.Bytes("example", original), ""); err == nil {
		t.Fatal("missing secret accepted")
	}
	t.Setenv("AUDITA_PROTOTYPE_KEY", "key")
	loaded, err := loadConfig(ck.Bytes("example", original), "", intOverride("transient_retries", 0), ck.Override{Path: ck.Keys("llm", "validation_profile"), Value: "proposal", Flag: "--validation-profile"})
	if err != nil {
		t.Fatal(err)
	}
	if loaded.Config.LLM.TransientRetries != 0 || loaded.Config.LLM.ValidationProfile != "proposal" {
		t.Fatal("explicit override lost")
	}
}

func TestSelectedOverlayConfiguration(t *testing.T) {
	t.Setenv("AUDITA_PROTOTYPE_KEY", "synthetic-test-secret")
	base, err := os.ReadFile("config.yaml")
	if err != nil {
		t.Fatal(err)
	}
	overlay, err := os.ReadFile("overlays/testing.yml")
	if err != nil {
		t.Fatal(err)
	}
	for _, tc := range []struct {
		name, base, overlay, selection, proposal, validation string
		overrides                                            []ck.Override
		invalid                                              bool
	}{
		{name: "omitted", base: string(base), overlay: string(overlay), proposal: "proposal", validation: "assessment"},
		{name: "empty", base: string(base), overlay: string(overlay), selection: "", proposal: "proposal", validation: "assessment"},
		{name: "default", base: string(base), overlay: string(overlay), selection: "default", proposal: "proposal", validation: "assessment"},
		{name: "testing", base: string(base), overlay: string(overlay), selection: "testing", proposal: "proposal-testing", validation: "assessment-testing"},
		{name: "proposal override", base: string(base), overlay: string(overlay), selection: "testing", proposal: "proposal", validation: "assessment-testing", overrides: []ck.Override{{Path: ck.Keys("llm", "proposal_profile"), Value: "proposal", Flag: "--proposal-profile"}}},
		{name: "validation override", base: string(base), overlay: string(overlay), selection: "testing", proposal: "proposal-testing", validation: "assessment", overrides: []ck.Override{{Path: ck.Keys("llm", "validation_profile"), Value: "assessment", Flag: "--validation-profile"}}},
		{name: "inherited overlay proposal", base: strings.Replace(string(base), "  validation_profile: assessment\n", "", 1), overlay: "llm:\n  proposal_profile: proposal-testing\n", selection: "testing", proposal: "proposal-testing", validation: "proposal-testing"},
		{name: "inherited override", base: strings.Replace(string(base), "  validation_profile: assessment\n", "", 1), overlay: "llm:\n  proposal_profile: proposal-testing\n", selection: "testing", proposal: "proposal", validation: "proposal", overrides: []ck.Override{{Path: ck.Keys("llm", "proposal_profile"), Value: "proposal", Flag: "--proposal-profile"}}},
		{name: "explicit base assessment", base: string(base), overlay: "llm:\n  proposal_profile: proposal-testing\n", selection: "testing", proposal: "proposal-testing", validation: "assessment"},
		{name: "empty overlay assessment", base: string(base), overlay: "llm:\n  validation_profile: ''\n", selection: "testing", invalid: true},
		{name: "null overlay assessment", base: string(base), overlay: "llm:\n  validation_profile: null\n", selection: "testing", invalid: true},
	} {
		t.Run(tc.name, func(t *testing.T) {
			dir := t.TempDir()
			if err := os.MkdirAll(filepath.Join(dir, "overlays"), 0700); err != nil {
				t.Fatal(err)
			}
			if err := os.WriteFile(filepath.Join(dir, "config.yaml"), []byte(tc.base), 0600); err != nil {
				t.Fatal(err)
			}
			if err := os.WriteFile(filepath.Join(dir, "overlays", "testing.yml"), []byte(tc.overlay), 0600); err != nil {
				t.Fatal(err)
			}
			loaded, err := loadConfig(ck.File(filepath.Join(dir, "config.yaml")), tc.selection, tc.overrides...)
			if tc.invalid {
				if err == nil {
					t.Fatal("expected invalid validation selection")
				}
				return
			}
			if err != nil {
				t.Fatal(err)
			}
			if got := loaded.Config.LLM; got.ProposalProfile != tc.proposal || got.ValidationProfile != tc.validation {
				t.Fatalf("roles = %q, %q; want %q, %q", got.ProposalProfile, got.ValidationProfile, tc.proposal, tc.validation)
			}
			for _, name := range []string{"proposal.yaml", "assessment.yaml", "proposal-testing.yaml", "assessment-testing.yaml"} {
				data, err := os.ReadFile(filepath.Join("profiles", name))
				if err != nil {
					t.Fatal(err)
				}
				if err := os.MkdirAll(filepath.Join(dir, "profiles"), 0700); err != nil {
					t.Fatal(err)
				}
				if err := os.WriteFile(filepath.Join(dir, "profiles", name), data, 0600); err != nil {
					t.Fatal(err)
				}
			}
			r, err := newRuntime(loaded.Config, dir, demonstrationClient())
			if err != nil {
				t.Fatal(err)
			}
			for role, selected := range map[string]string{"proposal": tc.proposal, "assessment": tc.validation} {
				profile := r.Profiles[role]
				if profile.ProfileID != selected || profile.EffectiveModelParams.BackendID != "offline" {
					t.Fatalf("%s resolved to %+v", role, profile)
				}
				models := map[string]string{"proposal": "synthetic-grammar", "assessment": "synthetic-review", "proposal-testing": "synthetic-grammar-testing", "assessment-testing": "synthetic-review-testing"}
				if profile.EffectiveModelParams.Model != models[selected] {
					t.Fatalf("%s model = %q", role, profile.EffectiveModelParams.Model)
				}
			}
		})
	}
}

func TestProfilesRespectSharedBackendBoundary(t *testing.T) {
	t.Setenv("AUDITA_PROTOTYPE_KEY", "synthetic-test-secret")
	loaded, err := loadConfig(ck.File("config.yaml"), "")
	if err != nil {
		t.Fatal(err)
	}
	for _, tc := range []struct{ name, profile string }{
		{"endpoint only", "id: custom\nendpoint: https://offline.invalid/v1\nmodel: example\n"},
		{"endpoint override", "id: custom\nbackend: offline\nendpoint: https://other.invalid/v1\nmodel: example\n"},
		{"competing secret lookup", "id: custom\nbackend: offline\nmodel: example\napi_key_env: OTHER_PROTOTYPE_KEY\n"},
	} {
		t.Run(tc.name, func(t *testing.T) {
			dir := t.TempDir()
			if err := os.WriteFile(dir+"/custom.yaml", []byte(tc.profile), 0600); err != nil {
				t.Fatal(err)
			}
			c := loaded.Config
			c.ProfilesDir = dir
			c.LLM.ProposalProfile = "custom"
			c.LLM.ValidationProfile = "custom"
			if _, err := newRuntime(c, ".", demonstrationClient()); err == nil {
				t.Fatal("profile bypass accepted")
			}
		})
	}
}