Configure correction repair profiles and budgets
This commit is contained in:
@@ -46,11 +46,13 @@ type GlossaryEntry struct {
|
||||
Extra map[string]any `yaml:",inline"`
|
||||
}
|
||||
type LLM struct {
|
||||
ProposalProfile *string `yaml:"proposal_profile"`
|
||||
ValidationProfile *string `yaml:"validation_profile"`
|
||||
TransientRetries int `yaml:"transient_retries"`
|
||||
RepairAttempts int `yaml:"repair_attempts"`
|
||||
OperationTimeout time.Duration `yaml:"operation_timeout"`
|
||||
ProposalProfile *string `yaml:"proposal_profile"`
|
||||
ValidationProfile *string `yaml:"validation_profile"`
|
||||
CorrectionRepairProfile *string `yaml:"correction_repair_profile"`
|
||||
CorrectionRepairAttempts int `yaml:"correction_repair_attempts"`
|
||||
TransientRetries int `yaml:"transient_retries"`
|
||||
RepairAttempts int `yaml:"repair_attempts"`
|
||||
OperationTimeout time.Duration `yaml:"operation_timeout"`
|
||||
}
|
||||
type Pipeline struct {
|
||||
Stages []Stage `yaml:"stages"`
|
||||
@@ -102,7 +104,7 @@ type Reporting struct {
|
||||
func defaults() Config {
|
||||
return Config{
|
||||
Version: 3,
|
||||
LLM: LLM{TransientRetries: 2, RepairAttempts: 2, OperationTimeout: 15 * time.Minute},
|
||||
LLM: LLM{TransientRetries: 2, RepairAttempts: 2, CorrectionRepairAttempts: 1, OperationTimeout: 15 * time.Minute},
|
||||
Pipeline: Pipeline{Stages: []Stage{{Module: "glossary"}, {Module: "homophones"}, {Module: "glossary"}, {Module: "spoken_word"}, {Module: "grammar"}}},
|
||||
Output: Output{Schema: "bare-segments"},
|
||||
Validation: Validation{MaxPromptTokens: DefaultValidationMaxPromptTokens},
|
||||
@@ -167,6 +169,8 @@ func validate(c *Config, v *ck.Validation) {
|
||||
}
|
||||
check(c.LLM.TransientRetries >= 0, ck.Keys("llm", "transient_retries"), "retry budget must be non-negative")
|
||||
check(c.LLM.RepairAttempts >= 0 && c.LLM.RepairAttempts <= 3, ck.Keys("llm", "repair_attempts"), "repair attempts must be between zero and three")
|
||||
check(c.LLM.CorrectionRepairAttempts >= 0, ck.Keys("llm", "correction_repair_attempts"), "correction repair attempts must be non-negative")
|
||||
check(c.LLM.CorrectionRepairProfile == nil || *c.LLM.CorrectionRepairProfile != "", ck.Keys("llm", "correction_repair_profile"), "correction repair profile must not be empty; use null to clear it")
|
||||
check(c.LLM.OperationTimeout > 0, ck.Keys("llm", "operation_timeout"), "operation timeout must be positive")
|
||||
check(c.LLM.ValidationProfile == nil || *c.LLM.ValidationProfile != "", ck.Keys("llm", "validation_profile"), "validation profile must not be empty; use null to clear it")
|
||||
validateSelections := func(values []string, path ck.Path) {
|
||||
@@ -190,6 +194,8 @@ func validate(c *Config, v *ck.Validation) {
|
||||
validateSelections(item.ProtectedTerms, path.Key("protected_terms"))
|
||||
check(item.ConfidenceThreshold >= 0 && item.ConfidenceThreshold <= 1, path.Key("confidence_threshold"), "confidence threshold must be between zero and one")
|
||||
check(item.LLM.ValidationProfile == nil || *item.LLM.ValidationProfile != "", path.Key("llm").Key("validation_profile"), "validation profile must not be empty; use null to clear it")
|
||||
check(item.LLM.CorrectionRepairAttempts >= 0, path.Key("llm").Key("correction_repair_attempts"), "correction repair attempts must be non-negative")
|
||||
check(item.LLM.CorrectionRepairProfile == nil || *item.LLM.CorrectionRepairProfile != "", path.Key("llm").Key("correction_repair_profile"), "correction repair profile must not be empty; use null to clear it")
|
||||
check(item.LLM.TransientRetries >= 0, path.Key("llm").Key("transient_retries"), "retry budget must be non-negative")
|
||||
check(item.LLM.RepairAttempts >= 0 && item.LLM.RepairAttempts <= 3, path.Key("llm").Key("repair_attempts"), "repair attempts must be between zero and three")
|
||||
check(item.LLM.OperationTimeout > 0, path.Key("llm").Key("operation_timeout"), "operation timeout must be positive")
|
||||
|
||||
@@ -28,6 +28,9 @@ func resolveSettings(c *Config, r *ck.Resolution) error {
|
||||
rootValidation := ck.Keys("llm", "validation_profile")
|
||||
rootValidationInfo, _ := r.EntryInfo(rootValidation)
|
||||
inheritedValidation := r.IsExplicit(rootValidation) && !rootValidationInfo.Null && c.LLM.ValidationProfile != nil
|
||||
rootRepair := ck.Keys("llm", "correction_repair_profile")
|
||||
rootRepairInfo, _ := r.EntryInfo(rootRepair)
|
||||
inheritedRepair := r.IsExplicit(rootRepair) && !rootRepairInfo.Null && c.LLM.CorrectionRepairProfile != nil
|
||||
for i := range c.Pipeline.Stages {
|
||||
item := &c.Pipeline.Stages[i]
|
||||
base := ck.Keys("pipeline", "stages").Index(i)
|
||||
@@ -52,9 +55,20 @@ func resolveSettings(c *Config, r *ck.Resolution) error {
|
||||
ck.Default(r, proposal, &item.LLM.ProposalProfile, cloneSelection(c.LLM.ProposalProfile), "inherit proposal profile", rootLLM.Key("proposal_profile"))
|
||||
ck.Default(r, llm.Key("transient_retries"), &item.LLM.TransientRetries, c.LLM.TransientRetries, "inherit retry budget", rootLLM.Key("transient_retries"))
|
||||
ck.Default(r, llm.Key("repair_attempts"), &item.LLM.RepairAttempts, c.LLM.RepairAttempts, "inherit repair budget", rootLLM.Key("repair_attempts"))
|
||||
ck.Default(r, llm.Key("correction_repair_attempts"), &item.LLM.CorrectionRepairAttempts, c.LLM.CorrectionRepairAttempts, "inherit correction repair budget", rootLLM.Key("correction_repair_attempts"))
|
||||
ck.Default(r, llm.Key("operation_timeout"), &item.LLM.OperationTimeout, c.LLM.OperationTimeout, "inherit operation timeout", rootLLM.Key("operation_timeout"))
|
||||
ck.Default(r, base.Key("validation").Key("max_prompt_tokens"), &item.Validation.MaxPromptTokens, c.Validation.MaxPromptTokens, "inherit validation budget", ck.Keys("validation", "max_prompt_tokens"))
|
||||
ck.Default(r, base.Key("context").Key("description"), &item.Context.Description, c.Context.Description, "inherit description", ck.Keys("context", "description"))
|
||||
repair := llm.Key("correction_repair_profile")
|
||||
repairInfo, _ := r.EntryInfo(repair)
|
||||
if r.IsExplicit(repair) && repairInfo.Null {
|
||||
item.LLM.CorrectionRepairProfile = cloneSelection(item.LLM.ProposalProfile)
|
||||
r.RecordDerivedDefault(repair, "correction repair defaults to proposal profile", proposal)
|
||||
} else if inheritedRepair {
|
||||
ck.Default(r, repair, &item.LLM.CorrectionRepairProfile, cloneSelection(c.LLM.CorrectionRepairProfile), "inherit correction repair profile", rootRepair)
|
||||
} else {
|
||||
ck.Default(r, repair, &item.LLM.CorrectionRepairProfile, cloneSelection(item.LLM.ProposalProfile), "correction repair defaults to proposal profile", proposal)
|
||||
}
|
||||
validation := llm.Key("validation_profile")
|
||||
info, _ := r.EntryInfo(validation)
|
||||
if r.IsExplicit(validation) && info.Null {
|
||||
@@ -66,6 +80,12 @@ func resolveSettings(c *Config, r *ck.Resolution) error {
|
||||
ck.Default(r, validation, &item.LLM.ValidationProfile, cloneSelection(item.LLM.ProposalProfile), "validation defaults to proposal profile", proposal)
|
||||
}
|
||||
}
|
||||
if r.IsExplicit(rootRepair) && rootRepairInfo.Null {
|
||||
c.LLM.CorrectionRepairProfile = cloneSelection(c.LLM.ProposalProfile)
|
||||
r.RecordDerivedDefault(rootRepair, "correction repair defaults to proposal profile", ck.Keys("llm", "proposal_profile"))
|
||||
} else {
|
||||
ck.Default(r, rootRepair, &c.LLM.CorrectionRepairProfile, cloneSelection(c.LLM.ProposalProfile), "correction repair defaults to proposal profile", ck.Keys("llm", "proposal_profile"))
|
||||
}
|
||||
if r.IsExplicit(rootValidation) && rootValidationInfo.Null {
|
||||
c.LLM.ValidationProfile = cloneSelection(c.LLM.ProposalProfile)
|
||||
r.RecordDerivedDefault(rootValidation, "validation defaults to proposal profile", ck.Keys("llm", "proposal_profile"))
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
ck "go.maximumdirect.net/configkit"
|
||||
@@ -158,3 +160,52 @@ pipeline:
|
||||
t.Fatalf("explicit null lost: %+v", info)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCorrectionRepairSettingsResolution(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, root, local, want string
|
||||
budget int
|
||||
}{
|
||||
{"default", "", "proposal_profile: B", "B", 1},
|
||||
{"inherit explicit", "correction_repair_profile: R\n correction_repair_attempts: 5", "proposal_profile: B", "R", 5},
|
||||
{"clear root selection and budget", "correction_repair_profile: R\n correction_repair_attempts: 5", "proposal_profile: B\n correction_repair_profile: null\n correction_repair_attempts: 0", "B", 0},
|
||||
{"root null", "correction_repair_profile: null", "proposal_profile: B", "B", 1},
|
||||
{"explicit selection", "correction_repair_profile: R", "correction_repair_profile: S", "S", 1},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
doc := "llm:\n proposal_profile: A\n"
|
||||
if tc.root != "" {
|
||||
doc += " " + tc.root + "\n"
|
||||
}
|
||||
doc += "pipeline:\n stages:\n - module: grammar\n llm:\n " + tc.local + "\n"
|
||||
result, err := loadResolution(t, doc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg := result.Config
|
||||
got := cfg.Pipeline.Stages[0].LLM
|
||||
if got.CorrectionRepairProfile == nil || *got.CorrectionRepairProfile != tc.want || got.CorrectionRepairAttempts != tc.budget {
|
||||
t.Fatalf("settings = %+v", got)
|
||||
}
|
||||
var effective bytes.Buffer
|
||||
if err := result.WriteEffective(&effective); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(effective.String(), "correction_repair_attempts:") || !strings.Contains(effective.String(), "correction_repair_profile:") {
|
||||
t.Fatalf("effective configuration: %s", effective.String())
|
||||
}
|
||||
if got.CorrectionRepairProfile == cfg.LLM.CorrectionRepairProfile || got.CorrectionRepairProfile == got.ProposalProfile {
|
||||
t.Fatal("profiles share storage")
|
||||
}
|
||||
path := ck.Keys("pipeline", "stages").Index(0).Key("llm").Key("correction_repair_profile")
|
||||
if _, ok := result.Explain(path); !ok {
|
||||
t.Fatal("missing resolution evidence")
|
||||
}
|
||||
})
|
||||
}
|
||||
for _, doc := range []string{"llm: {correction_repair_attempts: -1}", "pipeline:\n stages:\n - module: grammar\n llm: {correction_repair_attempts: -1}", "llm: {correction_repair_profile: ''}"} {
|
||||
if _, err := loadResolution(t, doc); err == nil {
|
||||
t.Errorf("accepted %s", doc)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user