Configure correction repair profiles and budgets

This commit is contained in:
2026-11-11 04:20:23 +00:00
parent 00848203d7
commit 9fd36001b4
9 changed files with 133 additions and 9 deletions

View File

@@ -38,6 +38,14 @@ func executePipeline(ctx context.Context, loaded *processInputs, prepared *prepa
return processResult{}, fmt.Errorf("%s validation client: %w", item.InstanceName, err)
}
item.ProposalClient, item.ValidationClient = proposal, validation
if item.Settings.LLM.CorrectionRepairAttempts > 0 {
repair, err := loaded.Runtime.NewRoleClient("correction_repair", *item.Settings.LLM.CorrectionRepairProfile, item.Settings.LLM, hooks)
if err != nil {
return processResult{}, fmt.Errorf("%s correction repair client: %w", item.InstanceName, err)
}
item.RepairClient = repair
item.CaptureProposalExchange = true
}
}
debugPath := prepared.DebugPath()
output, runErr := runner.New(modules.NewFactory()).Run(ctx, runner.RunInput{

View File

@@ -46,11 +46,13 @@ type GlossaryEntry struct {
Extra map[string]any `yaml:",inline"`
}
type LLM struct {
ProposalProfile *string `yaml:"proposal_profile"`
ValidationProfile *string `yaml:"validation_profile"`
TransientRetries int `yaml:"transient_retries"`
RepairAttempts int `yaml:"repair_attempts"`
OperationTimeout time.Duration `yaml:"operation_timeout"`
ProposalProfile *string `yaml:"proposal_profile"`
ValidationProfile *string `yaml:"validation_profile"`
CorrectionRepairProfile *string `yaml:"correction_repair_profile"`
CorrectionRepairAttempts int `yaml:"correction_repair_attempts"`
TransientRetries int `yaml:"transient_retries"`
RepairAttempts int `yaml:"repair_attempts"`
OperationTimeout time.Duration `yaml:"operation_timeout"`
}
type Pipeline struct {
Stages []Stage `yaml:"stages"`
@@ -102,7 +104,7 @@ type Reporting struct {
func defaults() Config {
return Config{
Version: 3,
LLM: LLM{TransientRetries: 2, RepairAttempts: 2, OperationTimeout: 15 * time.Minute},
LLM: LLM{TransientRetries: 2, RepairAttempts: 2, CorrectionRepairAttempts: 1, OperationTimeout: 15 * time.Minute},
Pipeline: Pipeline{Stages: []Stage{{Module: "glossary"}, {Module: "homophones"}, {Module: "glossary"}, {Module: "spoken_word"}, {Module: "grammar"}}},
Output: Output{Schema: "bare-segments"},
Validation: Validation{MaxPromptTokens: DefaultValidationMaxPromptTokens},
@@ -167,6 +169,8 @@ func validate(c *Config, v *ck.Validation) {
}
check(c.LLM.TransientRetries >= 0, ck.Keys("llm", "transient_retries"), "retry budget must be non-negative")
check(c.LLM.RepairAttempts >= 0 && c.LLM.RepairAttempts <= 3, ck.Keys("llm", "repair_attempts"), "repair attempts must be between zero and three")
check(c.LLM.CorrectionRepairAttempts >= 0, ck.Keys("llm", "correction_repair_attempts"), "correction repair attempts must be non-negative")
check(c.LLM.CorrectionRepairProfile == nil || *c.LLM.CorrectionRepairProfile != "", ck.Keys("llm", "correction_repair_profile"), "correction repair profile must not be empty; use null to clear it")
check(c.LLM.OperationTimeout > 0, ck.Keys("llm", "operation_timeout"), "operation timeout must be positive")
check(c.LLM.ValidationProfile == nil || *c.LLM.ValidationProfile != "", ck.Keys("llm", "validation_profile"), "validation profile must not be empty; use null to clear it")
validateSelections := func(values []string, path ck.Path) {
@@ -190,6 +194,8 @@ func validate(c *Config, v *ck.Validation) {
validateSelections(item.ProtectedTerms, path.Key("protected_terms"))
check(item.ConfidenceThreshold >= 0 && item.ConfidenceThreshold <= 1, path.Key("confidence_threshold"), "confidence threshold must be between zero and one")
check(item.LLM.ValidationProfile == nil || *item.LLM.ValidationProfile != "", path.Key("llm").Key("validation_profile"), "validation profile must not be empty; use null to clear it")
check(item.LLM.CorrectionRepairAttempts >= 0, path.Key("llm").Key("correction_repair_attempts"), "correction repair attempts must be non-negative")
check(item.LLM.CorrectionRepairProfile == nil || *item.LLM.CorrectionRepairProfile != "", path.Key("llm").Key("correction_repair_profile"), "correction repair profile must not be empty; use null to clear it")
check(item.LLM.TransientRetries >= 0, path.Key("llm").Key("transient_retries"), "retry budget must be non-negative")
check(item.LLM.RepairAttempts >= 0 && item.LLM.RepairAttempts <= 3, path.Key("llm").Key("repair_attempts"), "repair attempts must be between zero and three")
check(item.LLM.OperationTimeout > 0, path.Key("llm").Key("operation_timeout"), "operation timeout must be positive")

View File

@@ -28,6 +28,9 @@ func resolveSettings(c *Config, r *ck.Resolution) error {
rootValidation := ck.Keys("llm", "validation_profile")
rootValidationInfo, _ := r.EntryInfo(rootValidation)
inheritedValidation := r.IsExplicit(rootValidation) && !rootValidationInfo.Null && c.LLM.ValidationProfile != nil
rootRepair := ck.Keys("llm", "correction_repair_profile")
rootRepairInfo, _ := r.EntryInfo(rootRepair)
inheritedRepair := r.IsExplicit(rootRepair) && !rootRepairInfo.Null && c.LLM.CorrectionRepairProfile != nil
for i := range c.Pipeline.Stages {
item := &c.Pipeline.Stages[i]
base := ck.Keys("pipeline", "stages").Index(i)
@@ -52,9 +55,20 @@ func resolveSettings(c *Config, r *ck.Resolution) error {
ck.Default(r, proposal, &item.LLM.ProposalProfile, cloneSelection(c.LLM.ProposalProfile), "inherit proposal profile", rootLLM.Key("proposal_profile"))
ck.Default(r, llm.Key("transient_retries"), &item.LLM.TransientRetries, c.LLM.TransientRetries, "inherit retry budget", rootLLM.Key("transient_retries"))
ck.Default(r, llm.Key("repair_attempts"), &item.LLM.RepairAttempts, c.LLM.RepairAttempts, "inherit repair budget", rootLLM.Key("repair_attempts"))
ck.Default(r, llm.Key("correction_repair_attempts"), &item.LLM.CorrectionRepairAttempts, c.LLM.CorrectionRepairAttempts, "inherit correction repair budget", rootLLM.Key("correction_repair_attempts"))
ck.Default(r, llm.Key("operation_timeout"), &item.LLM.OperationTimeout, c.LLM.OperationTimeout, "inherit operation timeout", rootLLM.Key("operation_timeout"))
ck.Default(r, base.Key("validation").Key("max_prompt_tokens"), &item.Validation.MaxPromptTokens, c.Validation.MaxPromptTokens, "inherit validation budget", ck.Keys("validation", "max_prompt_tokens"))
ck.Default(r, base.Key("context").Key("description"), &item.Context.Description, c.Context.Description, "inherit description", ck.Keys("context", "description"))
repair := llm.Key("correction_repair_profile")
repairInfo, _ := r.EntryInfo(repair)
if r.IsExplicit(repair) && repairInfo.Null {
item.LLM.CorrectionRepairProfile = cloneSelection(item.LLM.ProposalProfile)
r.RecordDerivedDefault(repair, "correction repair defaults to proposal profile", proposal)
} else if inheritedRepair {
ck.Default(r, repair, &item.LLM.CorrectionRepairProfile, cloneSelection(c.LLM.CorrectionRepairProfile), "inherit correction repair profile", rootRepair)
} else {
ck.Default(r, repair, &item.LLM.CorrectionRepairProfile, cloneSelection(item.LLM.ProposalProfile), "correction repair defaults to proposal profile", proposal)
}
validation := llm.Key("validation_profile")
info, _ := r.EntryInfo(validation)
if r.IsExplicit(validation) && info.Null {
@@ -66,6 +80,12 @@ func resolveSettings(c *Config, r *ck.Resolution) error {
ck.Default(r, validation, &item.LLM.ValidationProfile, cloneSelection(item.LLM.ProposalProfile), "validation defaults to proposal profile", proposal)
}
}
if r.IsExplicit(rootRepair) && rootRepairInfo.Null {
c.LLM.CorrectionRepairProfile = cloneSelection(c.LLM.ProposalProfile)
r.RecordDerivedDefault(rootRepair, "correction repair defaults to proposal profile", ck.Keys("llm", "proposal_profile"))
} else {
ck.Default(r, rootRepair, &c.LLM.CorrectionRepairProfile, cloneSelection(c.LLM.ProposalProfile), "correction repair defaults to proposal profile", ck.Keys("llm", "proposal_profile"))
}
if r.IsExplicit(rootValidation) && rootValidationInfo.Null {
c.LLM.ValidationProfile = cloneSelection(c.LLM.ProposalProfile)
r.RecordDerivedDefault(rootValidation, "validation defaults to proposal profile", ck.Keys("llm", "proposal_profile"))

View File

@@ -1,9 +1,11 @@
package config
import (
"bytes"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
ck "go.maximumdirect.net/configkit"
@@ -158,3 +160,52 @@ pipeline:
t.Fatalf("explicit null lost: %+v", info)
}
}
func TestCorrectionRepairSettingsResolution(t *testing.T) {
for _, tc := range []struct {
name, root, local, want string
budget int
}{
{"default", "", "proposal_profile: B", "B", 1},
{"inherit explicit", "correction_repair_profile: R\n correction_repair_attempts: 5", "proposal_profile: B", "R", 5},
{"clear root selection and budget", "correction_repair_profile: R\n correction_repair_attempts: 5", "proposal_profile: B\n correction_repair_profile: null\n correction_repair_attempts: 0", "B", 0},
{"root null", "correction_repair_profile: null", "proposal_profile: B", "B", 1},
{"explicit selection", "correction_repair_profile: R", "correction_repair_profile: S", "S", 1},
} {
t.Run(tc.name, func(t *testing.T) {
doc := "llm:\n proposal_profile: A\n"
if tc.root != "" {
doc += " " + tc.root + "\n"
}
doc += "pipeline:\n stages:\n - module: grammar\n llm:\n " + tc.local + "\n"
result, err := loadResolution(t, doc)
if err != nil {
t.Fatal(err)
}
cfg := result.Config
got := cfg.Pipeline.Stages[0].LLM
if got.CorrectionRepairProfile == nil || *got.CorrectionRepairProfile != tc.want || got.CorrectionRepairAttempts != tc.budget {
t.Fatalf("settings = %+v", got)
}
var effective bytes.Buffer
if err := result.WriteEffective(&effective); err != nil {
t.Fatal(err)
}
if !strings.Contains(effective.String(), "correction_repair_attempts:") || !strings.Contains(effective.String(), "correction_repair_profile:") {
t.Fatalf("effective configuration: %s", effective.String())
}
if got.CorrectionRepairProfile == cfg.LLM.CorrectionRepairProfile || got.CorrectionRepairProfile == got.ProposalProfile {
t.Fatal("profiles share storage")
}
path := ck.Keys("pipeline", "stages").Index(0).Key("llm").Key("correction_repair_profile")
if _, ok := result.Explain(path); !ok {
t.Fatal("missing resolution evidence")
}
})
}
for _, doc := range []string{"llm: {correction_repair_attempts: -1}", "pipeline:\n stages:\n - module: grammar\n llm: {correction_repair_attempts: -1}", "llm: {correction_repair_profile: ''}"} {
if _, err := loadResolution(t, doc); err == nil {
t.Errorf("accepted %s", doc)
}
}
}

View File

@@ -48,6 +48,7 @@ type ModuleRunSpec struct {
CaptureProposalExchange bool `json:"-"`
ProposalClient StructuredLLMClient `json:"-"`
ValidationClient StructuredLLMClient `json:"-"`
RepairClient StructuredLLMClient `json:"-"`
}
// SectionMetadata captures chunk-level metadata without embedding full segment payloads.

View File

@@ -116,10 +116,17 @@ func CheckReadiness(ctx context.Context, cfg *config.Config, inspectProfile func
}
}
for index, item := range cfg.Pipeline.Stages {
for _, selection := range []struct {
selections := []struct {
role string
profile *string
}{{"proposal", item.LLM.ProposalProfile}, {"validation", item.LLM.ValidationProfile}} {
}{{"proposal", item.LLM.ProposalProfile}, {"validation", item.LLM.ValidationProfile}}
if item.LLM.CorrectionRepairAttempts > 0 {
selections = append(selections, struct {
role string
profile *string
}{"correction repair", item.LLM.CorrectionRepairProfile})
}
for _, selection := range selections {
if selection.profile == nil || *selection.profile == "" {
return fmt.Errorf("%s requires a %s profile", label(index, item), selection.role)
}

View File

@@ -2,6 +2,7 @@ package preflight
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
@@ -37,3 +38,26 @@ func TestEmptyPipelineNeedsNoProfile(t *testing.T) {
t.Fatal(err)
}
}
func TestCorrectionRepairReadinessDependsOnBudget(t *testing.T) {
proposal, repair := "proposal", "missing"
cfg := &config.Config{Pipeline: config.Pipeline{Stages: []config.Stage{{Module: "grammar", LLM: config.LLM{ProposalProfile: &proposal, ValidationProfile: &proposal, CorrectionRepairProfile: &repair}}}}}
inspect := func(_ context.Context, id string) error {
if id == repair {
return fmt.Errorf("unknown profile")
}
return nil
}
if err := CheckReadiness(context.Background(), cfg, inspect); err != nil {
t.Fatal(err)
}
cfg.Pipeline.Stages[0].LLM.CorrectionRepairAttempts = 1
output := filepath.Join(t.TempDir(), "transcript.json")
_, err := Prepare(context.Background(), Input{Config: cfg, Transcript: &schema.SourceTranscript{}, Output: diagnostics.OutputSelection{TranscriptPath: output}, InspectProfile: inspect})
if err == nil || !strings.Contains(err.Error(), "correction repair") {
t.Fatalf("readiness error: %v", err)
}
if _, err := os.Stat(output); !os.IsNotExist(err) {
t.Fatalf("output exists: %v", err)
}
}