eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Configure correction repair profiles and budgets

Browse Source
This commit is contained in:
Eric Rakestraw
2026-11-11 04:20:23 +00:00
parent 00848203d7
commit 9fd36001b4
9 changed files with 133 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
docs/config.md
View File

@@ -61,6 +61,8 @@ configuration.
| --- | --- | --- |
| `llm.proposal_profile` | unset | PromptKit profile for proposals; required for processing entries. |
| `llm.validation_profile` | effective proposal profile | Profile for model validation; null clears a selection before fallback. |
| `llm.correction_repair_profile` | effective proposal profile | Profile for correction repair; null clears an inherited selection before fallback. |
| `llm.correction_repair_attempts` | `1` | Nonnegative repair attempts per correction; zero disables correction repair. |
| `llm.transient_retries` | `2` | Additional transient attempts per generation. |
| `llm.repair_attempts` | `2` | Structural repairs, from zero through three. |
| `llm.operation_timeout` | `15m` | Positive duration per logical operation. |
@@ -77,6 +79,9 @@ configuration.
Processing profiles are selected explicitly; no profile or glossary is selected
automatically. The validation selection falls back after root and per-entry
settings are combined. Null does not disable validation. Numeric zero and false
settings are combined. An explicit root correction repair profile is inherited
by each entry unless overridden or cleared with null; otherwise the entry uses
its effective proposal profile. The repair profile is checked only when its
attempt budget is positive. Null does not disable validation. Numeric zero and false
retain their field meaning, subject to validation. Inspect recorded origins and
default dependencies using the [native commands](cli.md).

2
docs/roadmap/implementation.md
View File

@@ -139,6 +139,8 @@ validator responses remains [deferred](future.md#domain-aware-response-recovery)
## Stage 4 — Resolve configuration and wire the repair role
**Completed.**
1. In `internal/core/config`, add `llm.correction_repair_attempts` as a
nonnegative integer, default `1`, and `llm.correction_repair_profile` as an
optional profile selection. Reuse the stage `llm` block. Omitted stage

8
internal/cli/process_pipeline.go
View File

@@ -38,6 +38,14 @@ func executePipeline(ctx context.Context, loaded *processInputs, prepared *prepa
return processResult{}, fmt.Errorf("%s validation client: %w", item.InstanceName, err)
}
item.ProposalClient, item.ValidationClient = proposal, validation
if item.Settings.LLM.CorrectionRepairAttempts > 0 {
repair, err := loaded.Runtime.NewRoleClient("correction_repair", *item.Settings.LLM.CorrectionRepairProfile, item.Settings.LLM, hooks)
if err != nil {
return processResult{}, fmt.Errorf("%s correction repair client: %w", item.InstanceName, err)
}
item.RepairClient = repair
item.CaptureProposalExchange = true
}
}
debugPath := prepared.DebugPath()
output, runErr := runner.New(modules.NewFactory()).Run(ctx, runner.RunInput{

18
internal/core/config/config.go
View File

@@ -46,11 +46,13 @@ type GlossaryEntry struct {
Extra map[string]any `yaml:",inline"`
}
type LLM struct {
ProposalProfile *string `yaml:"proposal_profile"`
ValidationProfile *string `yaml:"validation_profile"`
TransientRetries int `yaml:"transient_retries"`
RepairAttempts int `yaml:"repair_attempts"`
OperationTimeout time.Duration `yaml:"operation_timeout"`
ProposalProfile *string `yaml:"proposal_profile"`
ValidationProfile *string `yaml:"validation_profile"`
CorrectionRepairProfile *string `yaml:"correction_repair_profile"`
CorrectionRepairAttempts int `yaml:"correction_repair_attempts"`
TransientRetries int `yaml:"transient_retries"`
RepairAttempts int `yaml:"repair_attempts"`
OperationTimeout time.Duration `yaml:"operation_timeout"`
}
type Pipeline struct {
Stages []Stage `yaml:"stages"`
@@ -102,7 +104,7 @@ type Reporting struct {
func defaults() Config {
return Config{
Version: 3,
LLM: LLM{TransientRetries: 2, RepairAttempts: 2, OperationTimeout: 15 * time.Minute},
LLM: LLM{TransientRetries: 2, RepairAttempts: 2, CorrectionRepairAttempts: 1, OperationTimeout: 15 * time.Minute},
Pipeline: Pipeline{Stages: []Stage{{Module: "glossary"}, {Module: "homophones"}, {Module: "glossary"}, {Module: "spoken_word"}, {Module: "grammar"}}},
Output: Output{Schema: "bare-segments"},
Validation: Validation{MaxPromptTokens: DefaultValidationMaxPromptTokens},
@@ -167,6 +169,8 @@ func validate(c *Config, v *ck.Validation) {
}
check(c.LLM.TransientRetries >= 0, ck.Keys("llm", "transient_retries"), "retry budget must be non-negative")
check(c.LLM.RepairAttempts >= 0 && c.LLM.RepairAttempts <= 3, ck.Keys("llm", "repair_attempts"), "repair attempts must be between zero and three")
check(c.LLM.CorrectionRepairAttempts >= 0, ck.Keys("llm", "correction_repair_attempts"), "correction repair attempts must be non-negative")
check(c.LLM.CorrectionRepairProfile == nil || *c.LLM.CorrectionRepairProfile != "", ck.Keys("llm", "correction_repair_profile"), "correction repair profile must not be empty; use null to clear it")
check(c.LLM.OperationTimeout > 0, ck.Keys("llm", "operation_timeout"), "operation timeout must be positive")
check(c.LLM.ValidationProfile == nil || *c.LLM.ValidationProfile != "", ck.Keys("llm", "validation_profile"), "validation profile must not be empty; use null to clear it")
validateSelections := func(values []string, path ck.Path) {
@@ -190,6 +194,8 @@ func validate(c *Config, v *ck.Validation) {
validateSelections(item.ProtectedTerms, path.Key("protected_terms"))
check(item.ConfidenceThreshold >= 0 && item.ConfidenceThreshold <= 1, path.Key("confidence_threshold"), "confidence threshold must be between zero and one")
check(item.LLM.ValidationProfile == nil || *item.LLM.ValidationProfile != "", path.Key("llm").Key("validation_profile"), "validation profile must not be empty; use null to clear it")
check(item.LLM.CorrectionRepairAttempts >= 0, path.Key("llm").Key("correction_repair_attempts"), "correction repair attempts must be non-negative")
check(item.LLM.CorrectionRepairProfile == nil || *item.LLM.CorrectionRepairProfile != "", path.Key("llm").Key("correction_repair_profile"), "correction repair profile must not be empty; use null to clear it")
check(item.LLM.TransientRetries >= 0, path.Key("llm").Key("transient_retries"), "retry budget must be non-negative")
check(item.LLM.RepairAttempts >= 0 && item.LLM.RepairAttempts <= 3, path.Key("llm").Key("repair_attempts"), "repair attempts must be between zero and three")
check(item.LLM.OperationTimeout > 0, path.Key("llm").Key("operation_timeout"), "operation timeout must be positive")

20
internal/core/config/resolve.go
View File

@@ -28,6 +28,9 @@ func resolveSettings(c *Config, r *ck.Resolution) error {
rootValidation := ck.Keys("llm", "validation_profile")
rootValidationInfo, _ := r.EntryInfo(rootValidation)
inheritedValidation := r.IsExplicit(rootValidation) && !rootValidationInfo.Null && c.LLM.ValidationProfile != nil
rootRepair := ck.Keys("llm", "correction_repair_profile")
rootRepairInfo, _ := r.EntryInfo(rootRepair)
inheritedRepair := r.IsExplicit(rootRepair) && !rootRepairInfo.Null && c.LLM.CorrectionRepairProfile != nil
for i := range c.Pipeline.Stages {
item := &c.Pipeline.Stages[i]
base := ck.Keys("pipeline", "stages").Index(i)
@@ -52,9 +55,20 @@ func resolveSettings(c *Config, r *ck.Resolution) error {
ck.Default(r, proposal, &item.LLM.ProposalProfile, cloneSelection(c.LLM.ProposalProfile), "inherit proposal profile", rootLLM.Key("proposal_profile"))
ck.Default(r, llm.Key("transient_retries"), &item.LLM.TransientRetries, c.LLM.TransientRetries, "inherit retry budget", rootLLM.Key("transient_retries"))
ck.Default(r, llm.Key("repair_attempts"), &item.LLM.RepairAttempts, c.LLM.RepairAttempts, "inherit repair budget", rootLLM.Key("repair_attempts"))
ck.Default(r, llm.Key("correction_repair_attempts"), &item.LLM.CorrectionRepairAttempts, c.LLM.CorrectionRepairAttempts, "inherit correction repair budget", rootLLM.Key("correction_repair_attempts"))
ck.Default(r, llm.Key("operation_timeout"), &item.LLM.OperationTimeout, c.LLM.OperationTimeout, "inherit operation timeout", rootLLM.Key("operation_timeout"))
ck.Default(r, base.Key("validation").Key("max_prompt_tokens"), &item.Validation.MaxPromptTokens, c.Validation.MaxPromptTokens, "inherit validation budget", ck.Keys("validation", "max_prompt_tokens"))
ck.Default(r, base.Key("context").Key("description"), &item.Context.Description, c.Context.Description, "inherit description", ck.Keys("context", "description"))
repair := llm.Key("correction_repair_profile")
repairInfo, _ := r.EntryInfo(repair)
if r.IsExplicit(repair) && repairInfo.Null {
item.LLM.CorrectionRepairProfile = cloneSelection(item.LLM.ProposalProfile)
r.RecordDerivedDefault(repair, "correction repair defaults to proposal profile", proposal)
} else if inheritedRepair {
ck.Default(r, repair, &item.LLM.CorrectionRepairProfile, cloneSelection(c.LLM.CorrectionRepairProfile), "inherit correction repair profile", rootRepair)
} else {
ck.Default(r, repair, &item.LLM.CorrectionRepairProfile, cloneSelection(item.LLM.ProposalProfile), "correction repair defaults to proposal profile", proposal)
}
validation := llm.Key("validation_profile")
info, _ := r.EntryInfo(validation)
if r.IsExplicit(validation) && info.Null {
@@ -66,6 +80,12 @@ func resolveSettings(c *Config, r *ck.Resolution) error {
ck.Default(r, validation, &item.LLM.ValidationProfile, cloneSelection(item.LLM.ProposalProfile), "validation defaults to proposal profile", proposal)
}
}
if r.IsExplicit(rootRepair) && rootRepairInfo.Null {
c.LLM.CorrectionRepairProfile = cloneSelection(c.LLM.ProposalProfile)
r.RecordDerivedDefault(rootRepair, "correction repair defaults to proposal profile", ck.Keys("llm", "proposal_profile"))
} else {
ck.Default(r, rootRepair, &c.LLM.CorrectionRepairProfile, cloneSelection(c.LLM.ProposalProfile), "correction repair defaults to proposal profile", ck.Keys("llm", "proposal_profile"))
}
if r.IsExplicit(rootValidation) && rootValidationInfo.Null {
c.LLM.ValidationProfile = cloneSelection(c.LLM.ProposalProfile)
r.RecordDerivedDefault(rootValidation, "validation defaults to proposal profile", ck.Keys("llm", "proposal_profile"))

51
internal/core/config/resolve_test.go
View File

@@ -1,9 +1,11 @@
package config
import (
"bytes"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
ck "go.maximumdirect.net/configkit"
@@ -158,3 +160,52 @@ pipeline:
t.Fatalf("explicit null lost: %+v", info)
}
}
func TestCorrectionRepairSettingsResolution(t *testing.T) {
for _, tc := range []struct {
name, root, local, want string
budget int
}{
{"default", "", "proposal_profile: B", "B", 1},
{"inherit explicit", "correction_repair_profile: R\n correction_repair_attempts: 5", "proposal_profile: B", "R", 5},
{"clear root selection and budget", "correction_repair_profile: R\n correction_repair_attempts: 5", "proposal_profile: B\n correction_repair_profile: null\n correction_repair_attempts: 0", "B", 0},
{"root null", "correction_repair_profile: null", "proposal_profile: B", "B", 1},
{"explicit selection", "correction_repair_profile: R", "correction_repair_profile: S", "S", 1},
} {
t.Run(tc.name, func(t *testing.T) {
doc := "llm:\n proposal_profile: A\n"
if tc.root != "" {
doc += " " + tc.root + "\n"
}
doc += "pipeline:\n stages:\n - module: grammar\n llm:\n " + tc.local + "\n"
result, err := loadResolution(t, doc)
if err != nil {
t.Fatal(err)
}
cfg := result.Config
got := cfg.Pipeline.Stages[0].LLM
if got.CorrectionRepairProfile == nil || *got.CorrectionRepairProfile != tc.want || got.CorrectionRepairAttempts != tc.budget {
t.Fatalf("settings = %+v", got)
}
var effective bytes.Buffer
if err := result.WriteEffective(&effective); err != nil {
t.Fatal(err)
}
if !strings.Contains(effective.String(), "correction_repair_attempts:") || !strings.Contains(effective.String(), "correction_repair_profile:") {
t.Fatalf("effective configuration: %s", effective.String())
}
if got.CorrectionRepairProfile == cfg.LLM.CorrectionRepairProfile || got.CorrectionRepairProfile == got.ProposalProfile {
t.Fatal("profiles share storage")
}
path := ck.Keys("pipeline", "stages").Index(0).Key("llm").Key("correction_repair_profile")
if _, ok := result.Explain(path); !ok {
t.Fatal("missing resolution evidence")
}
})
}
for _, doc := range []string{"llm: {correction_repair_attempts: -1}", "pipeline:\n stages:\n - module: grammar\n llm: {correction_repair_attempts: -1}", "llm: {correction_repair_profile: ''}"} {
if _, err := loadResolution(t, doc); err == nil {
t.Errorf("accepted %s", doc)
}
}
}

1
internal/framework/contracts/contracts.go
View File

@@ -48,6 +48,7 @@ type ModuleRunSpec struct {
CaptureProposalExchange bool `json:"-"`
ProposalClient StructuredLLMClient `json:"-"`
ValidationClient StructuredLLMClient `json:"-"`
RepairClient StructuredLLMClient `json:"-"`
}
// SectionMetadata captures chunk-level metadata without embedding full segment payloads.

11
internal/framework/preflight/preflight.go
View File

@@ -116,10 +116,17 @@ func CheckReadiness(ctx context.Context, cfg *config.Config, inspectProfile func
}
}
for index, item := range cfg.Pipeline.Stages {
for _, selection := range []struct {
selections := []struct {
role string
profile *string
}{{"proposal", item.LLM.ProposalProfile}, {"validation", item.LLM.ValidationProfile}} {
}{{"proposal", item.LLM.ProposalProfile}, {"validation", item.LLM.ValidationProfile}}
if item.LLM.CorrectionRepairAttempts > 0 {
selections = append(selections, struct {
role string
profile *string
}{"correction repair", item.LLM.CorrectionRepairProfile})
}
for _, selection := range selections {
if selection.profile == nil || *selection.profile == "" {
return fmt.Errorf("%s requires a %s profile", label(index, item), selection.role)
}

24
internal/framework/preflight/readiness_test.go
View File

@@ -2,6 +2,7 @@ package preflight
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
@@ -37,3 +38,26 @@ func TestEmptyPipelineNeedsNoProfile(t *testing.T) {
t.Fatal(err)
}
}
func TestCorrectionRepairReadinessDependsOnBudget(t *testing.T) {
proposal, repair := "proposal", "missing"
cfg := &config.Config{Pipeline: config.Pipeline{Stages: []config.Stage{{Module: "grammar", LLM: config.LLM{ProposalProfile: &proposal, ValidationProfile: &proposal, CorrectionRepairProfile: &repair}}}}}
inspect := func(_ context.Context, id string) error {
if id == repair {
return fmt.Errorf("unknown profile")
}
return nil
}
if err := CheckReadiness(context.Background(), cfg, inspect); err != nil {
t.Fatal(err)
}
cfg.Pipeline.Stages[0].LLM.CorrectionRepairAttempts = 1
output := filepath.Join(t.TempDir(), "transcript.json")
_, err := Prepare(context.Background(), Input{Config: cfg, Transcript: &schema.SourceTranscript{}, Output: diagnostics.OutputSelection{TranscriptPath: output}, InspectProfile: inspect})
if err == nil || !strings.Contains(err.Error(), "correction repair") {
t.Fatalf("readiness error: %v", err)
}
if _, err := os.Stat(output); !os.IsNotExist(err) {
t.Fatalf("output exists: %v", err)
}
}