Remove confidence scoring and refine correction prompts
This commit is contained in:
@@ -27,7 +27,6 @@ type Config struct {
|
||||
Validation Validation `yaml:"validation"`
|
||||
Chunking Chunking `yaml:"chunking"`
|
||||
Normalization Normalization `yaml:"normalization"`
|
||||
Thresholds Thresholds `yaml:"thresholds"`
|
||||
Context Context `yaml:"context"`
|
||||
Reporting Reporting `yaml:"reporting"`
|
||||
Diagnostics Diagnostics `yaml:"diagnostics"`
|
||||
@@ -58,14 +57,13 @@ type Pipeline struct {
|
||||
Stages []Stage `yaml:"stages"`
|
||||
}
|
||||
type Stage struct {
|
||||
Module string `yaml:"module"`
|
||||
ID *string `yaml:"id"`
|
||||
Glossaries []string `yaml:"glossaries"`
|
||||
ProtectedTerms []string `yaml:"protected_terms"`
|
||||
ConfidenceThreshold float64 `yaml:"confidence_threshold"`
|
||||
LLM LLM `yaml:"llm"`
|
||||
Validation Validation `yaml:"validation"`
|
||||
Context Context `yaml:"context"`
|
||||
Module string `yaml:"module"`
|
||||
ID *string `yaml:"id"`
|
||||
Glossaries []string `yaml:"glossaries"`
|
||||
ProtectedTerms []string `yaml:"protected_terms"`
|
||||
LLM LLM `yaml:"llm"`
|
||||
Validation Validation `yaml:"validation"`
|
||||
Context Context `yaml:"context"`
|
||||
}
|
||||
type Output struct {
|
||||
Schema string `yaml:"schema"`
|
||||
@@ -84,12 +82,6 @@ type Normalization struct {
|
||||
MaxSegmentDuration float64 `yaml:"max_segment_duration"`
|
||||
MaxSegmentTokens int `yaml:"max_segment_tokens"`
|
||||
}
|
||||
type Thresholds struct {
|
||||
Glossary float64 `yaml:"glossary"`
|
||||
Homophones float64 `yaml:"homophones"`
|
||||
SpokenWord float64 `yaml:"spoken_word"`
|
||||
Grammar float64 `yaml:"grammar"`
|
||||
}
|
||||
type Context struct {
|
||||
Description string `yaml:"description"`
|
||||
}
|
||||
@@ -110,7 +102,6 @@ func defaults() Config {
|
||||
Validation: Validation{MaxPromptTokens: DefaultValidationMaxPromptTokens},
|
||||
Chunking: Chunking{MaxSectionTokens: 8192, MinSectionTokens: 2048},
|
||||
Normalization: Normalization{MaxSegmentGap: 4, EllipsisGap: 3.5, MaxSegmentDuration: 60, MaxSegmentTokens: 2048},
|
||||
Thresholds: Thresholds{Glossary: .8, Homophones: .8, SpokenWord: .8, Grammar: .8},
|
||||
Reporting: Reporting{Enabled: true, StdoutDir: "."},
|
||||
Diagnostics: Diagnostics{WorkDir: "/tmp/audita"},
|
||||
}
|
||||
@@ -192,7 +183,6 @@ func validate(c *Config, v *ck.Validation) {
|
||||
}
|
||||
validateSelections(item.Glossaries, path.Key("glossaries"))
|
||||
validateSelections(item.ProtectedTerms, path.Key("protected_terms"))
|
||||
check(item.ConfidenceThreshold >= 0 && item.ConfidenceThreshold <= 1, path.Key("confidence_threshold"), "confidence threshold must be between zero and one")
|
||||
check(item.LLM.ValidationProfile == nil || *item.LLM.ValidationProfile != "", path.Key("llm").Key("validation_profile"), "validation profile must not be empty; use null to clear it")
|
||||
check(item.LLM.CorrectionRepairAttempts >= 0, path.Key("llm").Key("correction_repair_attempts"), "correction repair attempts must be non-negative")
|
||||
check(item.LLM.CorrectionRepairProfile == nil || *item.LLM.CorrectionRepairProfile != "", path.Key("llm").Key("correction_repair_profile"), "correction repair profile must not be empty; use null to clear it")
|
||||
@@ -211,8 +201,5 @@ func validate(c *Config, v *ck.Validation) {
|
||||
check(c.Normalization.EllipsisGap >= 0, ck.Keys("normalization", "ellipsis_gap"), "ellipsis gap must be non-negative")
|
||||
check(c.Normalization.MaxSegmentDuration > 0, ck.Keys("normalization", "max_segment_duration"), "maximum segment duration must be positive")
|
||||
check(c.Normalization.MaxSegmentTokens > 0, ck.Keys("normalization", "max_segment_tokens"), "maximum segment tokens must be positive")
|
||||
for name, threshold := range map[string]float64{"glossary": c.Thresholds.Glossary, "homophones": c.Thresholds.Homophones, "spoken_word": c.Thresholds.SpokenWord, "grammar": c.Thresholds.Grammar} {
|
||||
check(threshold >= 0 && threshold <= 1, ck.Keys("thresholds", name), "confidence threshold must be between zero and one")
|
||||
}
|
||||
check(len(strings.TrimSpace(c.Context.Description)) <= DefaultTranscriptDescriptionMaxChars, ck.Keys("context", "description"), fmt.Sprintf("description must be at most %d bytes after trimming", DefaultTranscriptDescriptionMaxChars))
|
||||
}
|
||||
|
||||
@@ -155,7 +155,6 @@ func TestSchemaConstraintsAndExplicitValues(t *testing.T) {
|
||||
{"removed debug", "diagnostics:\n debug: true\n", false},
|
||||
{"unknown module", "pipeline:\n stages:\n - module: unknown\n", false},
|
||||
{"invalid chunk limit", "chunking:\n target_sections: 0\n", false},
|
||||
{"invalid threshold", "thresholds:\n grammar: 1.1\n", false},
|
||||
{"null group", "llm: null\n", false},
|
||||
{"zero budgets", "llm:\n transient_retries: 0\n repair_attempts: 0\n", true},
|
||||
} {
|
||||
|
||||
@@ -36,19 +36,6 @@ func resolveSettings(c *Config, r *ck.Resolution) error {
|
||||
base := ck.Keys("pipeline", "stages").Index(i)
|
||||
defaultNames(r, base.Key("glossaries"), &item.Glossaries, c.Glossaries, "inherit root glossaries", glossaries)
|
||||
defaultNames(r, base.Key("protected_terms"), &item.ProtectedTerms, c.ProtectedTerms, "inherit root protection", protection)
|
||||
threshold := ck.Keys("thresholds", item.Module)
|
||||
var value float64
|
||||
switch item.Module {
|
||||
case "glossary":
|
||||
value = c.Thresholds.Glossary
|
||||
case "homophones":
|
||||
value = c.Thresholds.Homophones
|
||||
case "spoken_word":
|
||||
value = c.Thresholds.SpokenWord
|
||||
case "grammar":
|
||||
value = c.Thresholds.Grammar
|
||||
}
|
||||
ck.Default(r, base.Key("confidence_threshold"), &item.ConfidenceThreshold, value, "inherit module threshold", threshold)
|
||||
llm := base.Key("llm")
|
||||
rootLLM := ck.Keys("llm")
|
||||
proposal := llm.Key("proposal_profile")
|
||||
|
||||
@@ -102,7 +102,6 @@ func TestResolvedSettingsValidation(t *testing.T) {
|
||||
{"duplicate id", "pipeline:\n stages:\n - {module: grammar, id: same}\n - {module: grammar, id: same}\n"},
|
||||
{"empty root validation profile", "llm:\n validation_profile: ''\n"},
|
||||
{"empty local validation profile", "pipeline:\n stages:\n - module: grammar\n llm:\n validation_profile: ''\n"},
|
||||
{"invalid local threshold", "pipeline:\n stages:\n - {module: grammar, confidence_threshold: 1.1}\n"},
|
||||
{"invalid local timeout", "pipeline:\n stages:\n - module: grammar\n llm: {operation_timeout: 0s}\n"},
|
||||
{"invalid local batch budget", "pipeline:\n stages:\n - module: grammar\n validation: {max_prompt_tokens: 0}\n"},
|
||||
} {
|
||||
|
||||
Reference in New Issue
Block a user