Configure correction repair profiles and budgets

This commit is contained in:
2026-11-11 04:20:23 +00:00
parent 00848203d7
commit 9fd36001b4
9 changed files with 133 additions and 9 deletions

View File

@@ -46,11 +46,13 @@ type GlossaryEntry struct {
Extra map[string]any `yaml:",inline"`
}
type LLM struct {
ProposalProfile *string `yaml:"proposal_profile"`
ValidationProfile *string `yaml:"validation_profile"`
TransientRetries int `yaml:"transient_retries"`
RepairAttempts int `yaml:"repair_attempts"`
OperationTimeout time.Duration `yaml:"operation_timeout"`
ProposalProfile *string `yaml:"proposal_profile"`
ValidationProfile *string `yaml:"validation_profile"`
CorrectionRepairProfile *string `yaml:"correction_repair_profile"`
CorrectionRepairAttempts int `yaml:"correction_repair_attempts"`
TransientRetries int `yaml:"transient_retries"`
RepairAttempts int `yaml:"repair_attempts"`
OperationTimeout time.Duration `yaml:"operation_timeout"`
}
type Pipeline struct {
Stages []Stage `yaml:"stages"`
@@ -102,7 +104,7 @@ type Reporting struct {
func defaults() Config {
return Config{
Version: 3,
LLM: LLM{TransientRetries: 2, RepairAttempts: 2, OperationTimeout: 15 * time.Minute},
LLM: LLM{TransientRetries: 2, RepairAttempts: 2, CorrectionRepairAttempts: 1, OperationTimeout: 15 * time.Minute},
Pipeline: Pipeline{Stages: []Stage{{Module: "glossary"}, {Module: "homophones"}, {Module: "glossary"}, {Module: "spoken_word"}, {Module: "grammar"}}},
Output: Output{Schema: "bare-segments"},
Validation: Validation{MaxPromptTokens: DefaultValidationMaxPromptTokens},
@@ -167,6 +169,8 @@ func validate(c *Config, v *ck.Validation) {
}
check(c.LLM.TransientRetries >= 0, ck.Keys("llm", "transient_retries"), "retry budget must be non-negative")
check(c.LLM.RepairAttempts >= 0 && c.LLM.RepairAttempts <= 3, ck.Keys("llm", "repair_attempts"), "repair attempts must be between zero and three")
check(c.LLM.CorrectionRepairAttempts >= 0, ck.Keys("llm", "correction_repair_attempts"), "correction repair attempts must be non-negative")
check(c.LLM.CorrectionRepairProfile == nil || *c.LLM.CorrectionRepairProfile != "", ck.Keys("llm", "correction_repair_profile"), "correction repair profile must not be empty; use null to clear it")
check(c.LLM.OperationTimeout > 0, ck.Keys("llm", "operation_timeout"), "operation timeout must be positive")
check(c.LLM.ValidationProfile == nil || *c.LLM.ValidationProfile != "", ck.Keys("llm", "validation_profile"), "validation profile must not be empty; use null to clear it")
validateSelections := func(values []string, path ck.Path) {
@@ -190,6 +194,8 @@ func validate(c *Config, v *ck.Validation) {
validateSelections(item.ProtectedTerms, path.Key("protected_terms"))
check(item.ConfidenceThreshold >= 0 && item.ConfidenceThreshold <= 1, path.Key("confidence_threshold"), "confidence threshold must be between zero and one")
check(item.LLM.ValidationProfile == nil || *item.LLM.ValidationProfile != "", path.Key("llm").Key("validation_profile"), "validation profile must not be empty; use null to clear it")
check(item.LLM.CorrectionRepairAttempts >= 0, path.Key("llm").Key("correction_repair_attempts"), "correction repair attempts must be non-negative")
check(item.LLM.CorrectionRepairProfile == nil || *item.LLM.CorrectionRepairProfile != "", path.Key("llm").Key("correction_repair_profile"), "correction repair profile must not be empty; use null to clear it")
check(item.LLM.TransientRetries >= 0, path.Key("llm").Key("transient_retries"), "retry budget must be non-negative")
check(item.LLM.RepairAttempts >= 0 && item.LLM.RepairAttempts <= 3, path.Key("llm").Key("repair_attempts"), "repair attempts must be between zero and three")
check(item.LLM.OperationTimeout > 0, path.Key("llm").Key("operation_timeout"), "operation timeout must be positive")

View File

@@ -28,6 +28,9 @@ func resolveSettings(c *Config, r *ck.Resolution) error {
rootValidation := ck.Keys("llm", "validation_profile")
rootValidationInfo, _ := r.EntryInfo(rootValidation)
inheritedValidation := r.IsExplicit(rootValidation) && !rootValidationInfo.Null && c.LLM.ValidationProfile != nil
rootRepair := ck.Keys("llm", "correction_repair_profile")
rootRepairInfo, _ := r.EntryInfo(rootRepair)
inheritedRepair := r.IsExplicit(rootRepair) && !rootRepairInfo.Null && c.LLM.CorrectionRepairProfile != nil
for i := range c.Pipeline.Stages {
item := &c.Pipeline.Stages[i]
base := ck.Keys("pipeline", "stages").Index(i)
@@ -52,9 +55,20 @@ func resolveSettings(c *Config, r *ck.Resolution) error {
ck.Default(r, proposal, &item.LLM.ProposalProfile, cloneSelection(c.LLM.ProposalProfile), "inherit proposal profile", rootLLM.Key("proposal_profile"))
ck.Default(r, llm.Key("transient_retries"), &item.LLM.TransientRetries, c.LLM.TransientRetries, "inherit retry budget", rootLLM.Key("transient_retries"))
ck.Default(r, llm.Key("repair_attempts"), &item.LLM.RepairAttempts, c.LLM.RepairAttempts, "inherit repair budget", rootLLM.Key("repair_attempts"))
ck.Default(r, llm.Key("correction_repair_attempts"), &item.LLM.CorrectionRepairAttempts, c.LLM.CorrectionRepairAttempts, "inherit correction repair budget", rootLLM.Key("correction_repair_attempts"))
ck.Default(r, llm.Key("operation_timeout"), &item.LLM.OperationTimeout, c.LLM.OperationTimeout, "inherit operation timeout", rootLLM.Key("operation_timeout"))
ck.Default(r, base.Key("validation").Key("max_prompt_tokens"), &item.Validation.MaxPromptTokens, c.Validation.MaxPromptTokens, "inherit validation budget", ck.Keys("validation", "max_prompt_tokens"))
ck.Default(r, base.Key("context").Key("description"), &item.Context.Description, c.Context.Description, "inherit description", ck.Keys("context", "description"))
repair := llm.Key("correction_repair_profile")
repairInfo, _ := r.EntryInfo(repair)
if r.IsExplicit(repair) && repairInfo.Null {
item.LLM.CorrectionRepairProfile = cloneSelection(item.LLM.ProposalProfile)
r.RecordDerivedDefault(repair, "correction repair defaults to proposal profile", proposal)
} else if inheritedRepair {
ck.Default(r, repair, &item.LLM.CorrectionRepairProfile, cloneSelection(c.LLM.CorrectionRepairProfile), "inherit correction repair profile", rootRepair)
} else {
ck.Default(r, repair, &item.LLM.CorrectionRepairProfile, cloneSelection(item.LLM.ProposalProfile), "correction repair defaults to proposal profile", proposal)
}
validation := llm.Key("validation_profile")
info, _ := r.EntryInfo(validation)
if r.IsExplicit(validation) && info.Null {
@@ -66,6 +80,12 @@ func resolveSettings(c *Config, r *ck.Resolution) error {
ck.Default(r, validation, &item.LLM.ValidationProfile, cloneSelection(item.LLM.ProposalProfile), "validation defaults to proposal profile", proposal)
}
}
if r.IsExplicit(rootRepair) && rootRepairInfo.Null {
c.LLM.CorrectionRepairProfile = cloneSelection(c.LLM.ProposalProfile)
r.RecordDerivedDefault(rootRepair, "correction repair defaults to proposal profile", ck.Keys("llm", "proposal_profile"))
} else {
ck.Default(r, rootRepair, &c.LLM.CorrectionRepairProfile, cloneSelection(c.LLM.ProposalProfile), "correction repair defaults to proposal profile", ck.Keys("llm", "proposal_profile"))
}
if r.IsExplicit(rootValidation) && rootValidationInfo.Null {
c.LLM.ValidationProfile = cloneSelection(c.LLM.ProposalProfile)
r.RecordDerivedDefault(rootValidation, "validation defaults to proposal profile", ck.Keys("llm", "proposal_profile"))

View File

@@ -1,9 +1,11 @@
package config
import (
"bytes"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
ck "go.maximumdirect.net/configkit"
@@ -158,3 +160,52 @@ pipeline:
t.Fatalf("explicit null lost: %+v", info)
}
}
func TestCorrectionRepairSettingsResolution(t *testing.T) {
for _, tc := range []struct {
name, root, local, want string
budget int
}{
{"default", "", "proposal_profile: B", "B", 1},
{"inherit explicit", "correction_repair_profile: R\n correction_repair_attempts: 5", "proposal_profile: B", "R", 5},
{"clear root selection and budget", "correction_repair_profile: R\n correction_repair_attempts: 5", "proposal_profile: B\n correction_repair_profile: null\n correction_repair_attempts: 0", "B", 0},
{"root null", "correction_repair_profile: null", "proposal_profile: B", "B", 1},
{"explicit selection", "correction_repair_profile: R", "correction_repair_profile: S", "S", 1},
} {
t.Run(tc.name, func(t *testing.T) {
doc := "llm:\n proposal_profile: A\n"
if tc.root != "" {
doc += " " + tc.root + "\n"
}
doc += "pipeline:\n stages:\n - module: grammar\n llm:\n " + tc.local + "\n"
result, err := loadResolution(t, doc)
if err != nil {
t.Fatal(err)
}
cfg := result.Config
got := cfg.Pipeline.Stages[0].LLM
if got.CorrectionRepairProfile == nil || *got.CorrectionRepairProfile != tc.want || got.CorrectionRepairAttempts != tc.budget {
t.Fatalf("settings = %+v", got)
}
var effective bytes.Buffer
if err := result.WriteEffective(&effective); err != nil {
t.Fatal(err)
}
if !strings.Contains(effective.String(), "correction_repair_attempts:") || !strings.Contains(effective.String(), "correction_repair_profile:") {
t.Fatalf("effective configuration: %s", effective.String())
}
if got.CorrectionRepairProfile == cfg.LLM.CorrectionRepairProfile || got.CorrectionRepairProfile == got.ProposalProfile {
t.Fatal("profiles share storage")
}
path := ck.Keys("pipeline", "stages").Index(0).Key("llm").Key("correction_repair_profile")
if _, ok := result.Explain(path); !ok {
t.Fatal("missing resolution evidence")
}
})
}
for _, doc := range []string{"llm: {correction_repair_attempts: -1}", "pipeline:\n stages:\n - module: grammar\n llm: {correction_repair_attempts: -1}", "llm: {correction_repair_profile: ''}"} {
if _, err := loadResolution(t, doc); err == nil {
t.Errorf("accepted %s", doc)
}
}
}