213 lines
11 KiB
Go
213 lines
11 KiB
Go
package config
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/modulecatalog"
|
|
"gitea.maximumdirect.net/eric/audita/v3/internal/core/outputschema"
|
|
ck "go.maximumdirect.net/configkit"
|
|
)
|
|
|
|
const (
|
|
DefaultTranscriptDescriptionMaxChars = 500
|
|
DefaultValidationMaxPromptTokens = 2048
|
|
)
|
|
|
|
// Config contains application settings. Invocation paths are not settings.
|
|
type Config struct {
|
|
Version int `yaml:"version"`
|
|
Glossaries []string `yaml:"glossaries"`
|
|
ProtectedTerms []string `yaml:"protected_terms"`
|
|
GlossaryCollections map[string]GlossaryCollection `yaml:"glossary_collections"`
|
|
LLM LLM `yaml:"llm"`
|
|
Pipeline Pipeline `yaml:"pipeline"`
|
|
Output Output `yaml:"output"`
|
|
Validation Validation `yaml:"validation"`
|
|
Chunking Chunking `yaml:"chunking"`
|
|
Normalization Normalization `yaml:"normalization"`
|
|
Thresholds Thresholds `yaml:"thresholds"`
|
|
Context Context `yaml:"context"`
|
|
Reporting Reporting `yaml:"reporting"`
|
|
Diagnostics Diagnostics `yaml:"diagnostics"`
|
|
}
|
|
|
|
type GlossaryCollection struct {
|
|
Entries []GlossaryEntry `yaml:"entries"`
|
|
}
|
|
type GlossaryEntry struct {
|
|
ID string `yaml:"id"`
|
|
Name string `yaml:"name"`
|
|
Category string `yaml:"category"`
|
|
Description string `yaml:"description"`
|
|
Aliases []string `yaml:"aliases"`
|
|
Plural *string `yaml:"plural"`
|
|
Extra map[string]any `yaml:",inline"`
|
|
}
|
|
type LLM struct {
|
|
ProposalProfile *string `yaml:"proposal_profile"`
|
|
ValidationProfile *string `yaml:"validation_profile"`
|
|
TransientRetries int `yaml:"transient_retries"`
|
|
RepairAttempts int `yaml:"repair_attempts"`
|
|
OperationTimeout time.Duration `yaml:"operation_timeout"`
|
|
}
|
|
type Pipeline struct {
|
|
Stages []Stage `yaml:"stages"`
|
|
}
|
|
type Stage struct {
|
|
Module string `yaml:"module"`
|
|
ID *string `yaml:"id"`
|
|
Glossaries []string `yaml:"glossaries"`
|
|
ProtectedTerms []string `yaml:"protected_terms"`
|
|
ConfidenceThreshold float64 `yaml:"confidence_threshold"`
|
|
LLM LLM `yaml:"llm"`
|
|
Validation Validation `yaml:"validation"`
|
|
Context Context `yaml:"context"`
|
|
}
|
|
type Output struct {
|
|
Schema string `yaml:"schema"`
|
|
}
|
|
type Validation struct {
|
|
MaxPromptTokens int `yaml:"max_prompt_tokens"`
|
|
}
|
|
type Chunking struct {
|
|
TargetSections *int `yaml:"target_sections"`
|
|
MaxSectionTokens int `yaml:"max_section_tokens"`
|
|
MinSectionTokens int `yaml:"min_section_tokens"`
|
|
}
|
|
type Normalization struct {
|
|
MaxSegmentGap float64 `yaml:"max_segment_gap"`
|
|
EllipsisGap float64 `yaml:"ellipsis_gap"`
|
|
MaxSegmentDuration float64 `yaml:"max_segment_duration"`
|
|
MaxSegmentTokens int `yaml:"max_segment_tokens"`
|
|
}
|
|
type Thresholds struct {
|
|
Glossary float64 `yaml:"glossary"`
|
|
Homophones float64 `yaml:"homophones"`
|
|
SpokenWord float64 `yaml:"spoken_word"`
|
|
Grammar float64 `yaml:"grammar"`
|
|
}
|
|
type Context struct {
|
|
Description string `yaml:"description"`
|
|
}
|
|
type Diagnostics struct {
|
|
WorkDir string `yaml:"work_dir"`
|
|
}
|
|
type Reporting struct {
|
|
Enabled bool `yaml:"enabled"`
|
|
StdoutDir string `yaml:"stdout_dir"`
|
|
}
|
|
|
|
func defaults() Config {
|
|
return Config{
|
|
Version: 3,
|
|
LLM: LLM{TransientRetries: 2, RepairAttempts: 2, OperationTimeout: 15 * time.Minute},
|
|
Pipeline: Pipeline{Stages: []Stage{{Module: "glossary"}, {Module: "homophones"}, {Module: "glossary"}, {Module: "spoken_word"}, {Module: "grammar"}}},
|
|
Output: Output{Schema: "bare-segments"},
|
|
Validation: Validation{MaxPromptTokens: DefaultValidationMaxPromptTokens},
|
|
Chunking: Chunking{MaxSectionTokens: 8192, MinSectionTokens: 2048},
|
|
Normalization: Normalization{MaxSegmentGap: 4, EllipsisGap: 3.5, MaxSegmentDuration: 60, MaxSegmentTokens: 2048},
|
|
Thresholds: Thresholds{Glossary: .8, Homophones: .8, SpokenWord: .8, Grammar: .8},
|
|
Reporting: Reporting{Enabled: true, StdoutDir: "."},
|
|
Diagnostics: Diagnostics{WorkDir: "/tmp/audita"},
|
|
}
|
|
}
|
|
|
|
const glossaryImports = `config:
|
|
imports:
|
|
- path: glossary.yml
|
|
required: false
|
|
destination: [glossary_collections]
|
|
- dir: glossary
|
|
required: false
|
|
recursive: true
|
|
pattern: ["*.yml", "*.yaml"]
|
|
destination: [glossary_collections]
|
|
`
|
|
|
|
// Load resolves the selected main source and preserves native source evidence.
|
|
func Load(input ck.Input, overlay string, overrides ...ck.Override) (*ck.Result[Config], error) {
|
|
input.Imports = true
|
|
return ck.LoadLayered(input, ck.LayeredOptions[Config]{
|
|
Overlay: overlay, Overrides: overrides,
|
|
Underlays: []ck.Underlay{{Inputs: []ck.Input{{Source: ck.Bytes("Audita glossary imports", []byte(glossaryImports)), Imports: true, MainRelative: ck.MainRelative()}}}},
|
|
Definition: ck.Definition[Config]{
|
|
Defaults: defaults,
|
|
SequenceIdentities: []ck.SequenceIdentity{ck.SequenceIdentityMatching(ck.SequenceSelector{ck.SequenceLiteralKey("glossary_collections"), ck.SequenceAnyMappingKey(), ck.SequenceLiteralKey("entries")}, "id")},
|
|
Version: &ck.VersionRule{Path: ck.Keys("version"), Check: func(version ck.Version, v *ck.Validation) error {
|
|
if version.Present && (version.Null || version.Text != "3") {
|
|
v.Error(ck.Keys("version"), "use application configuration version 3; see docs/migrations/v3.md")
|
|
}
|
|
return nil
|
|
}},
|
|
Resolve: resolveSettings,
|
|
Validate: validate,
|
|
},
|
|
})
|
|
}
|
|
|
|
func validate(c *Config, v *ck.Validation) {
|
|
check := func(ok bool, path ck.Path, message string) {
|
|
if !ok {
|
|
v.Error(path, message)
|
|
}
|
|
}
|
|
for name, collection := range c.GlossaryCollections {
|
|
for i, entry := range collection.Entries {
|
|
path := ck.Keys("glossary_collections", name, "entries").Index(i)
|
|
check(entry.ID != "", path.Key("id"), "entry id must not be empty")
|
|
check(entry.Name != "", path.Key("name"), "entry name must not be empty")
|
|
check(entry.Category != "", path.Key("category"), "entry category must not be empty")
|
|
for index, alias := range entry.Aliases {
|
|
check(alias != "", path.Key("aliases").Index(index), "alias must not be empty")
|
|
}
|
|
check(entry.Plural == nil || *entry.Plural != "", path.Key("plural"), "plural must not be empty")
|
|
}
|
|
}
|
|
check(c.LLM.TransientRetries >= 0, ck.Keys("llm", "transient_retries"), "retry budget must be non-negative")
|
|
check(c.LLM.RepairAttempts >= 0 && c.LLM.RepairAttempts <= 3, ck.Keys("llm", "repair_attempts"), "repair attempts must be between zero and three")
|
|
check(c.LLM.OperationTimeout > 0, ck.Keys("llm", "operation_timeout"), "operation timeout must be positive")
|
|
check(c.LLM.ValidationProfile == nil || *c.LLM.ValidationProfile != "", ck.Keys("llm", "validation_profile"), "validation profile must not be empty; use null to clear it")
|
|
validateSelections := func(values []string, path ck.Path) {
|
|
for i, name := range values {
|
|
_, exists := c.GlossaryCollections[name]
|
|
check(exists, path.Index(i), "unknown glossary collection")
|
|
}
|
|
}
|
|
validateSelections(c.Glossaries, ck.Keys("glossaries"))
|
|
validateSelections(c.ProtectedTerms, ck.Keys("protected_terms"))
|
|
ids := make(map[string]bool)
|
|
for i, item := range c.Pipeline.Stages {
|
|
path := ck.Keys("pipeline", "stages").Index(i)
|
|
check(modulecatalog.IsSupported(item.Module), path.Key("module"), "unsupported module key")
|
|
if item.ID != nil {
|
|
check(*item.ID != "", path.Key("id"), "id must not be empty")
|
|
check(!ids[*item.ID], path.Key("id"), "id must be unique")
|
|
ids[*item.ID] = true
|
|
}
|
|
validateSelections(item.Glossaries, path.Key("glossaries"))
|
|
validateSelections(item.ProtectedTerms, path.Key("protected_terms"))
|
|
check(item.ConfidenceThreshold >= 0 && item.ConfidenceThreshold <= 1, path.Key("confidence_threshold"), "confidence threshold must be between zero and one")
|
|
check(item.LLM.ValidationProfile == nil || *item.LLM.ValidationProfile != "", path.Key("llm").Key("validation_profile"), "validation profile must not be empty; use null to clear it")
|
|
check(item.LLM.TransientRetries >= 0, path.Key("llm").Key("transient_retries"), "retry budget must be non-negative")
|
|
check(item.LLM.RepairAttempts >= 0 && item.LLM.RepairAttempts <= 3, path.Key("llm").Key("repair_attempts"), "repair attempts must be between zero and three")
|
|
check(item.LLM.OperationTimeout > 0, path.Key("llm").Key("operation_timeout"), "operation timeout must be positive")
|
|
check(item.Validation.MaxPromptTokens > 0, path.Key("validation").Key("max_prompt_tokens"), "validation prompt budget must be positive")
|
|
check(len(strings.TrimSpace(item.Context.Description)) <= DefaultTranscriptDescriptionMaxChars, path.Key("context").Key("description"), fmt.Sprintf("description must be at most %d bytes after trimming", DefaultTranscriptDescriptionMaxChars))
|
|
}
|
|
check(outputschema.IsSupported(c.Output.Schema), ck.Keys("output", "schema"), "unsupported output schema")
|
|
check(c.Validation.MaxPromptTokens > 0, ck.Keys("validation", "max_prompt_tokens"), "validation prompt budget must be positive")
|
|
check(c.Chunking.MaxSectionTokens > 0, ck.Keys("chunking", "max_section_tokens"), "maximum section tokens must be positive")
|
|
check(c.Chunking.MinSectionTokens > 0 && c.Chunking.MinSectionTokens <= c.Chunking.MaxSectionTokens, ck.Keys("chunking", "min_section_tokens"), "minimum section tokens must be positive and no greater than maximum")
|
|
check(c.Chunking.TargetSections == nil || *c.Chunking.TargetSections > 0, ck.Keys("chunking", "target_sections"), "target sections must be positive when set")
|
|
check(c.Normalization.MaxSegmentGap >= 0, ck.Keys("normalization", "max_segment_gap"), "maximum segment gap must be non-negative")
|
|
check(c.Normalization.EllipsisGap >= 0, ck.Keys("normalization", "ellipsis_gap"), "ellipsis gap must be non-negative")
|
|
check(c.Normalization.MaxSegmentDuration > 0, ck.Keys("normalization", "max_segment_duration"), "maximum segment duration must be positive")
|
|
check(c.Normalization.MaxSegmentTokens > 0, ck.Keys("normalization", "max_segment_tokens"), "maximum segment tokens must be positive")
|
|
for name, threshold := range map[string]float64{"glossary": c.Thresholds.Glossary, "homophones": c.Thresholds.Homophones, "spoken_word": c.Thresholds.SpokenWord, "grammar": c.Thresholds.Grammar} {
|
|
check(threshold >= 0 && threshold <= 1, ck.Keys("thresholds", name), "confidence threshold must be between zero and one")
|
|
}
|
|
check(len(strings.TrimSpace(c.Context.Description)) <= DefaultTranscriptDescriptionMaxChars, ck.Keys("context", "description"), fmt.Sprintf("description must be at most %d bytes after trimming", DefaultTranscriptDescriptionMaxChars))
|
|
}
|