Complete assembled validation and migrate remaining tests
This commit is contained in:
@@ -4,43 +4,17 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
ck "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/v3/internal/prompts"
|
||||
ck "go.maximumdirect.net/configkit"
|
||||
)
|
||||
|
||||
func runtimeAssemblyFixture(t *testing.T, selections string, documents map[string]string) (config.Config, string) {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
profileDir := filepath.Join(dir, "profiles")
|
||||
if err := os.Mkdir(profileDir, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
main := filepath.Join(dir, "config.yml")
|
||||
text := "version: 2\nprofiles_dir: profiles\nbackends:\n offline:\n endpoint: http://127.0.0.1:1/v1\n concurrency: 1\n queue_capacity: 0\nllm:\n" + selections
|
||||
if err := os.WriteFile(main, []byte(text), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for name, content := range documents {
|
||||
if err := os.WriteFile(filepath.Join(profileDir, name), []byte(content), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
loaded, err := config.Load(ck.File(main), "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return loaded.Config, main
|
||||
}
|
||||
|
||||
func copyEmbeddedPromptFilesystem(t *testing.T) fstest.MapFS {
|
||||
t.Helper()
|
||||
source := prompts.Filesystem()
|
||||
@@ -62,166 +36,18 @@ func copyEmbeddedPromptFilesystem(t *testing.T) fstest.MapFS {
|
||||
return files
|
||||
}
|
||||
|
||||
func TestNewRuntimeRequiresCompleteSelectedProfiles(t *testing.T) {
|
||||
validProposal := " backend: offline\n model: proposal-model\n"
|
||||
cases := []struct {
|
||||
name string
|
||||
selections string
|
||||
profiles string
|
||||
role string
|
||||
field string
|
||||
}{
|
||||
{
|
||||
name: "omitted proposal model with derived validation selection",
|
||||
selections: " proposal_profile: proposal\n",
|
||||
profiles: "profiles:\n proposal:\n backend: offline\n",
|
||||
role: "proposal",
|
||||
field: "model",
|
||||
},
|
||||
{
|
||||
name: "empty validation model",
|
||||
selections: " proposal_profile: proposal\n validation_profile: validation\n",
|
||||
profiles: "profiles:\n proposal:\n" + validProposal + " validation:\n backend: offline\n model: \"\"\n",
|
||||
role: "validation",
|
||||
field: "model",
|
||||
},
|
||||
{
|
||||
name: "null proposal model",
|
||||
selections: " proposal_profile: proposal\n",
|
||||
profiles: "profiles:\n proposal:\n backend: offline\n model: null\n",
|
||||
role: "proposal",
|
||||
field: "model",
|
||||
},
|
||||
{
|
||||
name: "cleared validation endpoint",
|
||||
selections: " proposal_profile: proposal\n validation_profile: validation\n",
|
||||
profiles: "profiles:\n proposal:\n" + validProposal + " validation:\n backend: offline\n endpoint: null\n model: validation-model\n",
|
||||
role: "validation",
|
||||
field: "endpoint",
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
cfg, main := runtimeAssemblyFixture(t, tc.selections, map[string]string{"definitions.yml": tc.profiles})
|
||||
_, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}})
|
||||
if err == nil || !strings.Contains(err.Error(), tc.role) || !strings.Contains(err.Error(), tc.field) {
|
||||
t.Fatalf("selected incomplete profile was not rejected for %s.%s: %v", tc.role, tc.field, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRuntimeAllowsUnselectedPartialProfiles(t *testing.T) {
|
||||
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{
|
||||
"definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n unfinished:\n backend: offline\n",
|
||||
})
|
||||
runtime, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if runtime.Profiles["proposal"].EffectiveModelParams.Model != "proposal-model" || runtime.Profiles["validation"].EffectiveModelParams.Model != "proposal-model" {
|
||||
t.Fatalf("role profiles were not resolved: %+v", runtime.Profiles)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNativeProfileInheritancePreservesControlsAndMergesProviderValues(t *testing.T) {
|
||||
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: \"child profile\"\n", map[string]string{
|
||||
"unrelated-name.yaml": `profiles:
|
||||
"parent~/shared":
|
||||
backend: offline
|
||||
temperature: 0.4
|
||||
top_p: 0.7
|
||||
extra_params:
|
||||
routing:
|
||||
retained: from-parent
|
||||
overridden: parent
|
||||
cleared: remove-me
|
||||
"child profile":
|
||||
extends: "parent~/shared"
|
||||
model: child-model
|
||||
extra_params:
|
||||
routing:
|
||||
overridden: child
|
||||
child_only: present
|
||||
cleared: null
|
||||
`,
|
||||
})
|
||||
runtime, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
profile := runtime.Profiles["proposal"]
|
||||
target := profile.EffectiveModelParams
|
||||
if profile.ProfileID != "child profile" || target.Model != "child-model" || target.Temperature != 0.4 || target.TopP != 0.7 {
|
||||
t.Fatalf("inherited profile target = %+v", target)
|
||||
}
|
||||
routing, ok := target.ExtraParams["routing"].(map[string]any)
|
||||
if !ok || routing["retained"] != "from-parent" || routing["overridden"] != "child" || routing["child_only"] != "present" {
|
||||
t.Fatalf("recursively inherited provider values = %#v", target.ExtraParams)
|
||||
}
|
||||
if value, exists := routing["cleared"]; !exists || value != nil {
|
||||
t.Fatalf("null did not replace the inherited provider value: %#v", routing)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMissingSelectedProfileDirectoryIsOperatorInputError(t *testing.T) {
|
||||
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{
|
||||
"definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n",
|
||||
})
|
||||
if err := os.RemoveAll(filepath.Join(filepath.Dir(main), cfg.ProfilesDir)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}})
|
||||
if err == nil || IsAssetError(err) {
|
||||
t.Fatalf("missing operator profile directory classification: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRuntimeRejectsMalformedUnselectedProfilesAndConflicts(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
document map[string]string
|
||||
selection string
|
||||
}{
|
||||
{
|
||||
name: "malformed unselected definition",
|
||||
document: map[string]string{"definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n unused:\n temperature: 9\n"},
|
||||
selection: " proposal_profile: proposal\n",
|
||||
},
|
||||
{
|
||||
name: "undefined selected backend",
|
||||
document: map[string]string{"definitions.yml": "profiles:\n proposal:\n backend: missing\n model: proposal-model\n"},
|
||||
selection: " proposal_profile: proposal\n",
|
||||
},
|
||||
{
|
||||
name: "conflicting directory contributions",
|
||||
document: map[string]string{
|
||||
"a.yml": "profiles:\n proposal:\n backend: offline\n model: first-model\n",
|
||||
"b.yml": "profiles:\n proposal:\n model: second-model\n",
|
||||
},
|
||||
selection: " proposal_profile: proposal\n",
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
cfg, main := runtimeAssemblyFixture(t, tc.selection, tc.document)
|
||||
_, err := NewRuntime(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}})
|
||||
if err == nil || IsAssetError(err) {
|
||||
t.Fatalf("operator definition classification: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeClassifiesEmbeddedAssetsAndHonorsCancellation(t *testing.T) {
|
||||
_, err := newRuntimeWithPromptFS(context.Background(), config.Config{}, "config.yml", nil, fstest.MapFS{})
|
||||
loaded, err := config.Load(ck.Input{Source: ck.NoSource()}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = newRuntimeWithPromptFS(context.Background(), loaded, nil, fstest.MapFS{})
|
||||
if err == nil || !IsAssetError(err) || !errors.Is(err, fs.ErrNotExist) {
|
||||
t.Fatalf("embedded discovery error was not preserved as an asset error: %v", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
_, err = newRuntimeWithPromptFS(ctx, config.Config{}, "config.yml", nil, fstest.MapFS{})
|
||||
_, err = newRuntimeWithPromptFS(ctx, loaded, nil, fstest.MapFS{})
|
||||
if !errors.Is(err, context.Canceled) || IsAssetError(err) {
|
||||
t.Fatalf("caller cancellation classification: %v", err)
|
||||
}
|
||||
@@ -229,12 +55,13 @@ func TestRuntimeClassifiesEmbeddedAssetsAndHonorsCancellation(t *testing.T) {
|
||||
|
||||
func TestBrokenEmbeddedPromptFailsEmbeddedOnlyConstruction(t *testing.T) {
|
||||
promptFS := copyEmbeddedPromptFilesystem(t)
|
||||
assetPath := path.Join(prompts.AssetRoot, prompts.PromptIDModuleGrammarProposal+".yaml")
|
||||
promptFS[assetPath] = &fstest.MapFile{Data: []byte("prompts:\n modules.grammar.proposal:\n v1:\n messages: []\n output:\n format: json\n validation_mode: json_schema\n schema_path: correction_set.v1.json\n")}
|
||||
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{
|
||||
"definitions.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\n",
|
||||
})
|
||||
_, err := newRuntimeWithPromptFS(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}}, promptFS)
|
||||
assetPath := path.Join(prompts.AssetRoot, "modules.grammar.proposal.yaml")
|
||||
promptFS[assetPath] = &fstest.MapFile{Data: []byte("collections:\n audita:\n prompts:\n - id: modules.grammar.proposal\n messages: []\n")}
|
||||
loaded, err := config.Load(ck.Input{Source: ck.NoSource()}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = newRuntimeWithPromptFS(context.Background(), loaded, nil, promptFS)
|
||||
if err == nil || !IsAssetError(err) {
|
||||
t.Fatalf("broken embedded prompt was not classified as an asset error: %v", err)
|
||||
}
|
||||
@@ -242,14 +69,24 @@ func TestBrokenEmbeddedPromptFailsEmbeddedOnlyConstruction(t *testing.T) {
|
||||
|
||||
func TestOperatorPromptCanRepairInvalidEmbeddedPrompt(t *testing.T) {
|
||||
promptFS := copyEmbeddedPromptFilesystem(t)
|
||||
assetPath := path.Join(prompts.AssetRoot, prompts.PromptIDModuleGrammarProposal+".yaml")
|
||||
promptFS[assetPath] = &fstest.MapFile{Data: []byte("prompts:\n modules.grammar.proposal:\n v1:\n messages: []\n output:\n format: json\n validation_mode: json_schema\n schema_path: correction_set.v1.json\n")}
|
||||
cfg, main := runtimeAssemblyFixture(t, " proposal_profile: proposal\n", map[string]string{
|
||||
"operator.yml": "profiles:\n proposal:\n backend: offline\n model: proposal-model\nprompts:\n modules.grammar.proposal:\n v1:\n messages:\n - role: system\n content: operator system\n - role: user\n content: operator user\n",
|
||||
})
|
||||
runtime, err := newRuntimeWithPromptFS(context.Background(), cfg, main, &http.Client{Transport: rejectingTransport{t}}, promptFS)
|
||||
assetPath := path.Join(prompts.AssetRoot, "modules.grammar.proposal.yaml")
|
||||
promptFS[assetPath] = &fstest.MapFile{Data: []byte("collections:\n audita:\n prompts:\n - id: modules.grammar.proposal\n messages: []\n")}
|
||||
dir := t.TempDir()
|
||||
for name, content := range map[string]string{
|
||||
"config.yml": "version: 3\n",
|
||||
"prompts.yml": "collections:\n audita:\n prompts:\n - id: modules.grammar.proposal\n messages:\n - role: user\n content: operator message\n output:\n format: text\n validation_mode: none\n",
|
||||
} {
|
||||
if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
loaded, err := config.Load(ck.Input{Source: ck.File(filepath.Join(dir, "config.yml"))}, "")
|
||||
if err != nil {
|
||||
t.Fatalf("valid final configuration was rejected with an invalid displaced embedded prompt: %v", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
runtime, err := newRuntimeWithPromptFS(context.Background(), loaded, nil, promptFS)
|
||||
if err != nil {
|
||||
t.Fatalf("valid final configuration rejected: %v", err)
|
||||
}
|
||||
if runtime.Engine == nil {
|
||||
t.Fatal("runtime engine was not retained")
|
||||
|
||||
Reference in New Issue
Block a user