212 lines
7.8 KiB
Go
212 lines
7.8 KiB
Go
package config
|
|
|
|
import (
|
|
"bytes"
|
|
"os"
|
|
"path/filepath"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
|
|
ck "go.maximumdirect.net/configkit"
|
|
)
|
|
|
|
func loadResolution(t *testing.T, text string, overrides ...ck.Override) (*ck.Result[Config], error) {
|
|
t.Helper()
|
|
file := filepath.Join(t.TempDir(), "config.yml")
|
|
if err := os.WriteFile(file, []byte(text), 0644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return Load(ck.Input{Source: ck.File(file)}, "", overrides...)
|
|
}
|
|
|
|
func TestResolutionCopiesRootOverridesAndHonorsClears(t *testing.T) {
|
|
document := `glossary_collections:
|
|
party: {entries: []}
|
|
spells: {entries: []}
|
|
glossaries: [party]
|
|
llm:
|
|
proposal_profile: A
|
|
repair_attempts: 2
|
|
context:
|
|
description: root
|
|
pipeline:
|
|
stages:
|
|
- module: glossary
|
|
glossaries: [party]
|
|
llm:
|
|
proposal_profile: B
|
|
repair_attempts: 0
|
|
context:
|
|
description: ""
|
|
- module: grammar
|
|
protected_terms: null
|
|
- module: grammar
|
|
glossaries: []
|
|
`
|
|
result, err := loadResolution(t, document, ck.Override{Path: ck.Keys("glossaries"), Value: []any{"spells"}, Flag: "--glossaries"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cfg := result.Config
|
|
if !reflect.DeepEqual(cfg.ProtectedTerms, []string{"spells"}) || !reflect.DeepEqual(cfg.Pipeline.Stages[0].Glossaries, []string{"party"}) || !reflect.DeepEqual(cfg.Pipeline.Stages[1].Glossaries, []string{"spells"}) || len(cfg.Pipeline.Stages[2].Glossaries) != 0 {
|
|
t.Fatalf("resolved: %+v", cfg)
|
|
}
|
|
if cfg.Pipeline.Stages[1].ProtectedTerms != nil || cfg.Pipeline.Stages[0].LLM.RepairAttempts != 0 || cfg.Pipeline.Stages[0].Context.Description != "" {
|
|
t.Fatalf("clears lost: %+v", cfg.Pipeline.Stages)
|
|
}
|
|
if *cfg.Pipeline.Stages[0].LLM.ValidationProfile != "B" || *cfg.Pipeline.Stages[1].LLM.ValidationProfile != "A" {
|
|
t.Fatalf("fallback: %+v", cfg.Pipeline.Stages)
|
|
}
|
|
cfg.Pipeline.Stages[0].ProtectedTerms[0] = "changed"
|
|
if cfg.ProtectedTerms[0] != "spells" || cfg.Pipeline.Stages[2].ProtectedTerms[0] != "spells" {
|
|
t.Fatal("selections share mutable storage")
|
|
}
|
|
info, ok := result.Explain(ck.Keys("pipeline", "stages").Index(0).Key("protected_terms"))
|
|
if !ok || len(info.History) == 0 {
|
|
t.Fatal("missing resolution evidence")
|
|
}
|
|
}
|
|
|
|
func TestValidationProfileFallback(t *testing.T) {
|
|
for _, tc := range []struct{ name, root, local, want string }{
|
|
{"proposal override", "", "proposal_profile: B", "B"},
|
|
{"inherited selection", "validation_profile: V", "proposal_profile: B", "V"},
|
|
{"explicit null", "validation_profile: V", "proposal_profile: B\n validation_profile: null", "B"},
|
|
{"root null", "validation_profile: null", "proposal_profile: B", "B"},
|
|
{"stage selection", "", "validation_profile: W", "W"},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
doc := "llm:\n proposal_profile: A\n"
|
|
if tc.root != "" {
|
|
doc += " " + tc.root + "\n"
|
|
}
|
|
doc += "pipeline:\n stages:\n - module: grammar\n llm:\n " + tc.local + "\n"
|
|
result, err := loadResolution(t, doc)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
value := result.Config.Pipeline.Stages[0].LLM.ValidationProfile
|
|
if value == nil || *value != tc.want {
|
|
t.Fatalf("profile=%v want=%s", value, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestResolvedSettingsValidation(t *testing.T) {
|
|
for _, tc := range []struct{ name, text string }{
|
|
{"unknown root collection", "glossaries: [missing]\n"},
|
|
{"unknown local collection", "pipeline:\n stages:\n - module: grammar\n protected_terms: [missing]\n"},
|
|
{"empty id", "pipeline:\n stages:\n - {module: grammar, id: ''}\n"},
|
|
{"duplicate id", "pipeline:\n stages:\n - {module: grammar, id: same}\n - {module: grammar, id: same}\n"},
|
|
{"empty root validation profile", "llm:\n validation_profile: ''\n"},
|
|
{"empty local validation profile", "pipeline:\n stages:\n - module: grammar\n llm:\n validation_profile: ''\n"},
|
|
{"invalid local threshold", "pipeline:\n stages:\n - {module: grammar, confidence_threshold: 1.1}\n"},
|
|
{"invalid local timeout", "pipeline:\n stages:\n - module: grammar\n llm: {operation_timeout: 0s}\n"},
|
|
{"invalid local batch budget", "pipeline:\n stages:\n - module: grammar\n validation: {max_prompt_tokens: 0}\n"},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
if _, err := loadResolution(t, tc.text); err == nil {
|
|
t.Fatal("invalid settings accepted")
|
|
}
|
|
})
|
|
}
|
|
if _, err := loadResolution(t, "pipeline:\n stages:\n - {module: grammar, id: 'a/b not slug'}\n"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestResolutionRecordsFieldDependenciesAndExplicitNull(t *testing.T) {
|
|
doc := `glossary_collections:
|
|
party: {entries: []}
|
|
glossaries: [party]
|
|
llm:
|
|
proposal_profile: A
|
|
pipeline:
|
|
stages:
|
|
- module: grammar
|
|
llm:
|
|
proposal_profile: B
|
|
validation_profile: null
|
|
`
|
|
result, err := loadResolution(t, doc)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
base := ck.Keys("pipeline", "stages").Index(0)
|
|
for _, tc := range []struct{ path, dependency ck.Path }{
|
|
{ck.Keys("protected_terms"), ck.Keys("glossaries")},
|
|
{base.Key("glossaries"), ck.Keys("glossaries")},
|
|
{base.Key("llm").Key("validation_profile"), base.Key("llm").Key("proposal_profile")},
|
|
} {
|
|
found := false
|
|
for _, change := range result.Metadata().History(tc.path) {
|
|
if change.Origin.Operation != ck.ChangeDerivedDefault {
|
|
continue
|
|
}
|
|
for _, dependency := range change.Dependencies {
|
|
if dependency.String() == tc.dependency.String() {
|
|
found = true
|
|
}
|
|
}
|
|
}
|
|
if !found {
|
|
t.Errorf("missing %s dependency on %s", tc.path.String(), tc.dependency.String())
|
|
}
|
|
}
|
|
info, ok := result.Metadata().Lookup(base.Key("llm").Key("validation_profile"))
|
|
if !ok || !info.Explicit {
|
|
t.Fatalf("explicit null lost: %+v", info)
|
|
}
|
|
}
|
|
|
|
func TestCorrectionRepairSettingsResolution(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name, root, local, want string
|
|
budget int
|
|
}{
|
|
{"default", "", "proposal_profile: B", "B", 1},
|
|
{"inherit explicit", "correction_repair_profile: R\n correction_repair_attempts: 5", "proposal_profile: B", "R", 5},
|
|
{"clear root selection and budget", "correction_repair_profile: R\n correction_repair_attempts: 5", "proposal_profile: B\n correction_repair_profile: null\n correction_repair_attempts: 0", "B", 0},
|
|
{"root null", "correction_repair_profile: null", "proposal_profile: B", "B", 1},
|
|
{"explicit selection", "correction_repair_profile: R", "correction_repair_profile: S", "S", 1},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
doc := "llm:\n proposal_profile: A\n"
|
|
if tc.root != "" {
|
|
doc += " " + tc.root + "\n"
|
|
}
|
|
doc += "pipeline:\n stages:\n - module: grammar\n llm:\n " + tc.local + "\n"
|
|
result, err := loadResolution(t, doc)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cfg := result.Config
|
|
got := cfg.Pipeline.Stages[0].LLM
|
|
if got.CorrectionRepairProfile == nil || *got.CorrectionRepairProfile != tc.want || got.CorrectionRepairAttempts != tc.budget {
|
|
t.Fatalf("settings = %+v", got)
|
|
}
|
|
var effective bytes.Buffer
|
|
if err := result.WriteEffective(&effective); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !strings.Contains(effective.String(), "correction_repair_attempts:") || !strings.Contains(effective.String(), "correction_repair_profile:") {
|
|
t.Fatalf("effective configuration: %s", effective.String())
|
|
}
|
|
if got.CorrectionRepairProfile == cfg.LLM.CorrectionRepairProfile || got.CorrectionRepairProfile == got.ProposalProfile {
|
|
t.Fatal("profiles share storage")
|
|
}
|
|
path := ck.Keys("pipeline", "stages").Index(0).Key("llm").Key("correction_repair_profile")
|
|
if _, ok := result.Explain(path); !ok {
|
|
t.Fatal("missing resolution evidence")
|
|
}
|
|
})
|
|
}
|
|
for _, doc := range []string{"llm: {correction_repair_attempts: -1}", "pipeline:\n stages:\n - module: grammar\n llm: {correction_repair_attempts: -1}", "llm: {correction_repair_profile: ''}"} {
|
|
if _, err := loadResolution(t, doc); err == nil {
|
|
t.Errorf("accepted %s", doc)
|
|
}
|
|
}
|
|
}
|