Resolve producer modes turn allowances cache controls and shared workers

This commit is contained in:
2026-11-21 22:49:41 +00:00
parent cc14a7a5cc
commit 40fbcc61b1
26 changed files with 655 additions and 1405 deletions

View File

@@ -3,19 +3,9 @@ package config
import (
"fmt"
"path/filepath"
"strings"
)
// DefaultChunkPlanRoot resolves the existing per-user chunk-plan cache root.
func DefaultChunkPlanRoot(userCacheDir func() (string, error)) (string, error) {
return defaultCacheFamilyRoot(userCacheDir, "chunk-plans")
}
func DefaultCheckpointRoot(userCacheDir func() (string, error)) (string, error) {
return defaultCacheFamilyRoot(userCacheDir, "checkpoints")
}
func defaultCacheFamilyRoot(userCacheDir func() (string, error), family string) (string, error) {
func DefaultCacheRoot(userCacheDir func() (string, error)) (string, error) {
if userCacheDir == nil {
return "", fmt.Errorf("user cache directory resolver must not be nil")
}
@@ -23,9 +13,8 @@ func defaultCacheFamilyRoot(userCacheDir func() (string, error), family string)
if err != nil {
return "", fmt.Errorf("resolve user cache directory: %w", err)
}
root = strings.TrimSpace(root)
if root == "" {
return "", fmt.Errorf("user cache directory must not be empty")
}
return filepath.Join(filepath.Clean(root), "notarius", family), nil
return filepath.Join(root, "notarius"), nil
}

View File

@@ -0,0 +1,21 @@
package config
import (
"errors"
"path/filepath"
"testing"
)
func TestDefaultCacheRootUsesOneApplicationDirectory(t *testing.T) {
root, err := DefaultCacheRoot(func() (string, error) { return "/tmp/user-cache", nil })
if err != nil {
t.Fatal(err)
}
if root != filepath.Join("/tmp/user-cache", "notarius") {
t.Fatalf("root = %q", root)
}
failure := errors.New("cache home unavailable")
if _, err := DefaultCacheRoot(func() (string, error) { return "", failure }); !errors.Is(err, failure) {
t.Fatalf("lost cache discovery error: %v", err)
}
}

View File

@@ -35,7 +35,7 @@ config:
steps:
- id: first
references: {source: {path: ./first.txt}}
artifacts: {first: {extract: {module: extract, retries: 4}}}
artifacts: {first: {extract: {module: extract, max_additional_turns: 4}}}
`)
write("two/pipeline.yml", `pipelines:
session:
@@ -156,7 +156,7 @@ pipelines:
input: {module: input}
steps:
- id: first
artifacts: {old: {extract: {module: extract, retries: 4}}}
artifacts: {old: {extract: {module: extract, max_additional_turns: 4}}}
- id: second
artifacts: {second: {extract: {module: extract}}}
child:

View File

@@ -5,60 +5,38 @@ import (
)
type Config struct {
Pipelines map[string]pipeline.PipelineProfile `json:"pipelines"`
Concurrency ConcurrencyConfig `json:"concurrency"`
Output OutputConfig `json:"output"`
Cache CacheConfig `json:"cache"`
Debug DebugConfig `json:"debug"`
Pipelines map[string]pipeline.PipelineProfile `json:"pipelines"`
Concurrency ConcurrencyConfig `json:"concurrency"`
Output OutputConfig `json:"output"`
CacheDirectory string `json:"cache_directory"`
Debug DebugConfig `json:"debug"`
}
type ConcurrencyConfig struct {
TotalLLM int `json:"total_llm"`
StageWorkers map[string]int `json:"stage_workers"`
Workers int `json:"workers"`
}
const (
defaultLLMConcurrency = 16
)
const defaultWorkers = 64
type OutputConfig struct {
Directory string `json:"directory"`
}
type CacheConfig struct {
ChunkPlans ChunkPlanCacheConfig `json:"chunk_plans"`
Checkpoints CheckpointCacheConfig `json:"checkpoints"`
}
type ChunkPlanCacheConfig struct {
Directory string `json:"directory,omitempty"`
Mode pipeline.ChunkCacheMode `json:"mode"`
}
type CheckpointCacheConfig struct {
Enabled bool `json:"enabled"`
Directory string `json:"directory,omitempty"`
}
type DebugConfig struct {
Directory string `json:"directory"`
}
func Default() Config {
return Config{
Pipelines: map[string]pipeline.PipelineProfile{},
Concurrency: ConcurrencyConfig{
TotalLLM: defaultLLMConcurrency,
StageWorkers: map[string]int{"extract": defaultLLMConcurrency},
},
Output: OutputConfig{Directory: "./notarius-output"},
Cache: CacheConfig{ChunkPlans: ChunkPlanCacheConfig{Mode: pipeline.ChunkCacheAuto}},
Debug: DebugConfig{Directory: "./notarius-debug"},
Pipelines: map[string]pipeline.PipelineProfile{},
Concurrency: ConcurrencyConfig{Workers: defaultWorkers},
Output: OutputConfig{Directory: "./notarius-output"},
Debug: DebugConfig{Directory: "/tmp/notarius-debug"},
}
}
func cloneConfig(in Config) Config {
out := in
out.Concurrency.StageWorkers = cloneIntMap(in.Concurrency.StageWorkers)
out.Pipelines = make(map[string]pipeline.PipelineProfile, len(in.Pipelines))
for key, profile := range in.Pipelines {
out.Pipelines[key] = clonePipelineProfile(profile)
@@ -66,35 +44,11 @@ func cloneConfig(in Config) Config {
return out
}
func cloneIntMap(in map[string]int) map[string]int {
if len(in) == 0 {
return nil
}
out := make(map[string]int, len(in))
for key, value := range in {
out[key] = value
}
return out
}
func (c Config) withDerivedStageWorkers() Config {
c = cloneConfig(c)
if _, exists := c.Concurrency.StageWorkers["extract"]; !exists {
if c.Concurrency.StageWorkers == nil {
c.Concurrency.StageWorkers = make(map[string]int)
}
c.Concurrency.StageWorkers["extract"] = c.Concurrency.TotalLLM
}
return c
}
func clonePipelineProfile(in pipeline.PipelineProfile) pipeline.PipelineProfile {
out := in
out.ValidationPolicy = cloneValidationPolicyOverride(in.ValidationPolicy)
if in.StructuredOutputRepairAttempts != nil {
value := *in.StructuredOutputRepairAttempts
out.StructuredOutputRepairAttempts = &value
}
out.Mode = cloneValue(in.Mode)
out.MaxAdditionalTurns = cloneValue(in.MaxAdditionalTurns)
out.Cache = cloneValue(in.Cache)
out.Input = cloneModuleBinding(in.Input)
out.Chunk = cloneModuleBinding(in.Chunk)
out.Output = cloneModuleBinding(in.Output)
@@ -175,11 +129,9 @@ func cloneReferenceSource(in pipeline.ReferenceSource) pipeline.ReferenceSource
func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding {
out := in
out.ValidationPolicy = cloneValidationPolicyOverride(in.ValidationPolicy)
if in.StructuredOutputRepairAttempts != nil {
value := *in.StructuredOutputRepairAttempts
out.StructuredOutputRepairAttempts = &value
}
out.Mode = cloneValue(in.Mode)
out.MaxAdditionalTurns = cloneValue(in.MaxAdditionalTurns)
out.Cache = cloneValue(in.Cache)
if len(in.Options) > 0 {
out.Options = cloneOptions(in.Options)
}
@@ -188,26 +140,6 @@ func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding {
return out
}
func cloneValidationPolicyOverride(in *pipeline.ValidationPolicyOverride) *pipeline.ValidationPolicyOverride {
if in == nil {
return nil
}
out := *in
if in.ProducerStructuralFailure != nil {
value := *in.ProducerStructuralFailure
out.ProducerStructuralFailure = &value
}
if in.SemanticRejection != nil {
value := *in.SemanticRejection
out.SemanticRejection = &value
}
if in.ValidatorFailure != nil {
value := *in.ValidatorFailure
out.ValidatorFailure = &value
}
return &out
}
func cloneValidatorOverride(in pipeline.ValidatorOverride) pipeline.ValidatorOverride {
out := pipeline.ValidatorOverride{Set: in.Set}
if len(in.Validators) > 0 {
@@ -244,3 +176,11 @@ func cloneOptionValue(value any) any {
return typed
}
}
func cloneValue[T any](value *T) *T {
if value == nil {
return nil
}
copy := *value
return &copy
}

View File

@@ -9,25 +9,26 @@ import (
// deliberately separate from Config so YAML presence and composition concerns
// do not reach runtime pipeline types.
type Document struct {
Version string `yaml:"version"`
PromptKit promptkit.Settings `yaml:"promptkit"`
Pipelines map[string]PipelineDocument `yaml:"pipelines,omitempty"`
Concurrency *ConcurrencyDocument `yaml:"concurrency,omitempty"`
Output *OutputDocument `yaml:"output,omitempty"`
Cache *CacheDocument `yaml:"cache,omitempty"`
Debug *DebugDocument `yaml:"debug,omitempty"`
Version string `yaml:"version"`
PromptKit promptkit.Settings `yaml:"promptkit"`
Pipelines map[string]PipelineDocument `yaml:"pipelines,omitempty"`
Concurrency *ConcurrencyDocument `yaml:"concurrency,omitempty"`
Output *OutputDocument `yaml:"output,omitempty"`
CacheDirectory *string `yaml:"cache_directory,omitempty"`
Debug *DebugDocument `yaml:"debug,omitempty"`
}
type PipelineDocument struct {
LLMProfile *string `yaml:"llm_profile,omitempty"`
StructuredOutputRepairAttempts *int `yaml:"structured_output_repair_attempts,omitempty"`
ValidationPolicy *ValidationPolicyDocument `yaml:"validation_policy,omitempty"`
Input BindingDocument `yaml:"input"`
Chunk *BindingDocument `yaml:"chunk,omitempty"`
Artifacts map[string]ArtifactLaneDocument `yaml:"artifacts,omitempty"`
Steps []PipelineStepDocument `yaml:"steps,omitempty"`
Output *BindingDocument `yaml:"output,omitempty"`
References map[string]ReferenceDocument `yaml:"references,omitempty"`
Mode *pipeline.ExecutionMode `yaml:"mode,omitempty"`
MaxAdditionalTurns *int `yaml:"max_additional_turns,omitempty"`
Cache *pipeline.CacheMode `yaml:"cache,omitempty"`
LLMProfile *string `yaml:"llm_profile,omitempty"`
Input BindingDocument `yaml:"input"`
Chunk *BindingDocument `yaml:"chunk,omitempty"`
Artifacts map[string]ArtifactLaneDocument `yaml:"artifacts,omitempty"`
Steps []PipelineStepDocument `yaml:"steps,omitempty"`
Output *BindingDocument `yaml:"output,omitempty"`
References map[string]ReferenceDocument `yaml:"references,omitempty"`
}
type PipelineStepDocument struct {
@@ -48,22 +49,20 @@ type ArtifactLaneDocument struct {
// deliberately finite subset so nested validator-only fields are rejected by
// ConfigKit's strict decoder.
type BindingDocument struct {
Module string `yaml:"module"`
LLMProfile *string `yaml:"llm_profile,omitempty"`
StructuredOutputRepairAttempts *int `yaml:"structured_output_repair_attempts,omitempty"`
ValidationPolicy *ValidationPolicyDocument `yaml:"validation_policy,omitempty"`
Retries int `yaml:"retries,omitempty"`
Options map[string]any `yaml:"options,omitempty"`
References map[string]ReferenceDocument `yaml:"references,omitempty"`
Validators []ValidatorDocument `yaml:"validators,omitempty"`
Mode *pipeline.ExecutionMode `yaml:"mode,omitempty"`
MaxAdditionalTurns *int `yaml:"max_additional_turns,omitempty"`
Cache *pipeline.CacheMode `yaml:"cache,omitempty"`
Module string `yaml:"module"`
LLMProfile *string `yaml:"llm_profile,omitempty"`
Options map[string]any `yaml:"options,omitempty"`
References map[string]ReferenceDocument `yaml:"references,omitempty"`
Validators []ValidatorDocument `yaml:"validators,omitempty"`
}
type ValidatorDocument struct {
Module string `yaml:"module"`
LLMProfile *string `yaml:"llm_profile,omitempty"`
StructuredOutputRepairAttempts *int `yaml:"structured_output_repair_attempts,omitempty"`
Retries int `yaml:"retries,omitempty"`
Options map[string]any `yaml:"options,omitempty"`
Module string `yaml:"module"`
LLMProfile *string `yaml:"llm_profile,omitempty"`
Options map[string]any `yaml:"options,omitempty"`
}
// ReferenceDocument is the explicit version 6 union. Later document
@@ -79,36 +78,14 @@ type ArtifactReferenceDocument struct {
Lane string `yaml:"lane"`
}
type ValidationPolicyDocument struct {
ProducerStructuralFailure *pipeline.ProducerStructuralFailureAction `yaml:"producer_structural_failure,omitempty"`
SemanticRejection *pipeline.SemanticRejectionAction `yaml:"semantic_rejection,omitempty"`
ValidatorFailure *pipeline.ValidatorFailureAction `yaml:"validator_failure,omitempty"`
}
type ConcurrencyDocument struct {
TotalLLM *int `yaml:"total_llm,omitempty"`
StageWorkers map[string]int `yaml:"stage_workers,omitempty"`
Workers *int `yaml:"workers,omitempty"`
}
type OutputDocument struct {
Directory *string `yaml:"directory,omitempty"`
}
type CacheDocument struct {
ChunkPlans *ChunkPlanCacheDocument `yaml:"chunk_plans,omitempty"`
Checkpoints *CheckpointCacheDocument `yaml:"checkpoints,omitempty"`
}
type ChunkPlanCacheDocument struct {
Directory *string `yaml:"directory,omitempty"`
Mode *string `yaml:"mode,omitempty"`
}
type CheckpointCacheDocument struct {
Enabled *bool `yaml:"enabled,omitempty"`
Directory *string `yaml:"directory,omitempty"`
}
type DebugDocument struct {
Directory *string `yaml:"directory,omitempty"`
}

View File

@@ -16,16 +16,9 @@ func normalizeDocumentPaths(document *Document, resolution *configkit.Resolution
if document.Output != nil {
normalizeDocumentPath(resolution, state, configkit.Keys("output", "directory"), &document.Output.Directory, false, "resolved output directory")
}
if document.Cache != nil {
if document.Cache.ChunkPlans != nil {
normalizeDocumentPath(resolution, state, configkit.Keys("cache", "chunk_plans", "directory"), &document.Cache.ChunkPlans.Directory, true, "resolved chunk-plan cache directory")
}
if document.Cache.Checkpoints != nil {
normalizeDocumentPath(resolution, state, configkit.Keys("cache", "checkpoints", "directory"), &document.Cache.Checkpoints.Directory, true, "resolved checkpoint cache directory")
}
}
normalizeDocumentPath(resolution, state, configkit.Keys("cache_directory"), &document.CacheDirectory, true, "resolved cache directory")
if document.Debug != nil {
normalizeDocumentPath(resolution, state, configkit.Keys("debug", "directory"), &document.Debug.Directory, false, "resolved debug directory")
normalizeDocumentPath(resolution, state, configkit.Keys("debug", "directory"), &document.Debug.Directory, true, "resolved debug directory")
}
for _, id := range sortedPipelineDocumentKeys(document.Pipelines) {
profile := document.Pipelines[id]

View File

@@ -242,15 +242,15 @@ func TestDocumentPathsKeepEmptyCacheDirectoriesAndAbsolutePaths(t *testing.T) {
root := t.TempDir()
configPath := filepath.Join(root, "notarius.yml")
abs := filepath.Join(t.TempDir(), "outside.txt")
if err := os.WriteFile(configPath, []byte("version: 6\ncache:\n chunk_plans: {directory: ''}\n checkpoints: {directory: ''}\npipelines:\n main:\n input: {module: input}\n references:\n outside: {path: "+abs+"}\n"), 0o600); err != nil {
if err := os.WriteFile(configPath, []byte("version: 6\ncache_directory: ''\npipelines:\n main:\n input: {module: input}\n references:\n outside: {path: "+abs+"}\n"), 0o600); err != nil {
t.Fatal(err)
}
loaded, err := loadConfig(configPath, LoadOptions{})
if err != nil {
t.Fatalf("loadConfig() error = %v", err)
}
if loaded.Config.Cache.ChunkPlans.Directory != "" || loaded.Config.Cache.Checkpoints.Directory != "" {
t.Fatalf("cache directories = %#v", loaded.Config.Cache)
if loaded.Config.CacheDirectory == "" {
t.Fatalf("cache directories = %#v", loaded.Config.CacheDirectory)
}
if got := loaded.Config.Pipelines["main"].References["outside"].Path; got != abs {
t.Fatalf("absolute reference = %q", got)

View File

@@ -142,13 +142,13 @@ func TestDocumentDefinitionReturnsIndependentDefaults(t *testing.T) {
t.Fatal(err)
}
first.Config.Pipelines["changed"] = PipelineDocument{}
first.Config.Concurrency.StageWorkers["extract"] = 3
*first.Config.Concurrency.Workers = 3
second, err := loadDocumentSource(configkit.Bytes("second.yml", []byte("version: 6\n")), LoadOptions{})
if err != nil {
t.Fatal(err)
}
if len(second.Config.Pipelines) != 0 || second.Config.Concurrency.StageWorkers["extract"] != defaultLLMConcurrency || len(second.Config.Concurrency.StageWorkers) != 1 {
if len(second.Config.Pipelines) != 0 || *second.Config.Concurrency.Workers != defaultWorkers {
t.Fatalf("defaults aliased across loads: %#v", second.Config)
}
}

View File

@@ -2,12 +2,14 @@ package config
import (
"fmt"
"os"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
type ResolveInput struct {
CacheOverride *pipeline.CacheMode
PipelineID string
Only []string
Catalog pipeline.ModuleCatalog
@@ -26,7 +28,14 @@ type EffectiveConfig struct {
}
func (c Config) Resolve(input ResolveInput) (EffectiveConfig, error) {
c = c.withDerivedStageWorkers()
c = cloneConfig(c)
if c.CacheDirectory == "" {
var err error
c.CacheDirectory, err = DefaultCacheRoot(os.UserCacheDir)
if err != nil {
return EffectiveConfig{}, err
}
}
if err := c.Validate(); err != nil {
return EffectiveConfig{}, err
}
@@ -45,6 +54,7 @@ func (c Config) Resolve(input ResolveInput) (EffectiveConfig, error) {
resolved, err := pipeline.ResolvePipeline(profile, pipeline.ResolveOptions{
Only: input.Only,
CacheOverride: cloneValue(input.CacheOverride),
LLMProfileOverride: input.LLMProfileOverride,
ReferenceOverrides: append([]pipeline.ReferenceBinding(nil), input.ReferenceOverrides...),
ReferenceUnbinds: append([]pipeline.ReferenceUnbind(nil), input.ReferenceUnbinds...),

View File

@@ -41,12 +41,12 @@ func TestPipelineInheritancePreservesAndUpsertsSteps(t *testing.T) {
pipelines:
parent:
input: {module: input}
structured_output_repair_attempts: 2
max_additional_turns: 2
steps:
- id: prepare
artifacts:
score:
extract: {module: extract, retries: 2}
extract: {module: extract, max_additional_turns: 2}
- id: summarize
artifacts:
notes:
@@ -54,7 +54,7 @@ pipelines:
child:
extends: parent
llm_profile: local
structured_output_repair_attempts: 0` + test.childSteps + "\n"
max_additional_turns: 0` + test.childSteps + "\n"
loaded, err := loadConfigSource(configkit.Bytes("inheritance.yml", []byte(source)), LoadOptions{})
if err != nil {
t.Fatalf("loadConfigSource() error = %v", err)
@@ -66,10 +66,10 @@ pipelines:
t.Fatalf("parent/child inputs = %#v / %#v", parent.Input, child.Input)
}
if child.LLMProfile != "local" ||
child.StructuredOutputRepairAttempts == nil || *child.StructuredOutputRepairAttempts != 0 {
child.MaxAdditionalTurns == nil || *child.MaxAdditionalTurns != 0 {
t.Fatalf("child overrides = %#v", child)
}
if parent.StructuredOutputRepairAttempts == nil || *parent.StructuredOutputRepairAttempts != 2 {
if parent.MaxAdditionalTurns == nil || *parent.MaxAdditionalTurns != 2 {
t.Fatalf("parent settings changed: %#v", parent)
}
if len(child.Steps) != len(test.wantStepIDs) {
@@ -116,20 +116,20 @@ pipelines:
input: {module: input}
artifacts:
base:
extract: {module: extract, retries: 2}
extract: {module: extract, max_additional_turns: 2}
derived:
extends: base
extract: {retries: 4}
extract: {max_additional_turns: 4}
`)), LoadOptions{})
if err != nil {
t.Fatalf("loadConfigSource() error = %v", err)
}
lanes := loaded.Config.Pipelines["session"].Artifacts
if lanes["base"].Extract.Module != "extract" || lanes["base"].Extract.Retries != 2 {
if lanes["base"].Extract.Module != "extract" || *lanes["base"].Extract.MaxAdditionalTurns != 2 {
t.Fatalf("base lane = %#v", lanes["base"])
}
if lanes["derived"].Extract.Module != "extract" || lanes["derived"].Extract.Retries != 4 {
if lanes["derived"].Extract.Module != "extract" || *lanes["derived"].Extract.MaxAdditionalTurns != 4 {
t.Fatalf("derived lane = %#v", lanes["derived"])
}
if _, err := loaded.Config.Resolve(ResolveInput{PipelineID: "session", Catalog: effectiveCatalog(t)}); err != nil {
@@ -144,7 +144,7 @@ func TestInheritedPresenceOverridesRemainExplicit(t *testing.T) {
pipelines:
parent:
input: {module: input}
structured_output_repair_attempts: 3
max_additional_turns: 3
chunk:
module: generic
validators: []
@@ -155,7 +155,7 @@ pipelines:
validators: [{module: configured-a}]
child:
extends: parent
structured_output_repair_attempts: 0
max_additional_turns: 0
artifacts:
score:
extract:
@@ -178,11 +178,11 @@ pipelines:
parent.Artifacts["score"].Extract.Validators.Validators[0].Module != "configured-a" {
t.Fatalf("parent extraction validators = %#v", parent.Artifacts["score"].Extract.Validators)
}
if parent.StructuredOutputRepairAttempts == nil || *parent.StructuredOutputRepairAttempts != 3 {
t.Fatalf("parent repair budget = %#v", parent.StructuredOutputRepairAttempts)
if parent.MaxAdditionalTurns == nil || *parent.MaxAdditionalTurns != 3 {
t.Fatalf("parent turn allowance = %#v", parent.MaxAdditionalTurns)
}
if child.StructuredOutputRepairAttempts == nil || *child.StructuredOutputRepairAttempts != 0 {
t.Fatalf("child zero repair budget = %#v", child.StructuredOutputRepairAttempts)
if child.MaxAdditionalTurns == nil || *child.MaxAdditionalTurns != 0 {
t.Fatalf("child zero turn allowance = %#v", child.MaxAdditionalTurns)
}
extract := child.Artifacts["score"].Extract
if !extract.Validators.Set || extract.Validators.Validators == nil || len(extract.Validators.Validators) != 0 {
@@ -197,8 +197,8 @@ pipelines:
t.Fatalf("Resolve(child) error = %v", err)
}
lanes := resolved.ResolvedPipeline.AllArtifactLanes()
if len(lanes) != 1 || lanes[0].Extract.StructuredOutputRepairAttempts == nil || *lanes[0].Extract.StructuredOutputRepairAttempts != 0 {
t.Fatalf("resolved child repair budget = %#v", lanes)
if len(lanes) != 1 || lanes[0].Extract.MaxAdditionalTurns == nil || *lanes[0].Extract.MaxAdditionalTurns != 0 {
t.Fatalf("resolved child turn allowance = %#v", lanes)
}
if !resolved.ResolvedPipeline.Chunk.Validators.Set || len(resolved.ResolvedPipeline.Chunk.Validators.Validators) != 0 ||
!lanes[0].Extract.Validators.Set || len(lanes[0].Extract.Validators.Validators) != 0 {
@@ -234,42 +234,6 @@ pipelines:
}
}
func TestInheritedAndDirectNullRepairBudgetsRemainInvalid(t *testing.T) {
t.Parallel()
for _, test := range []struct {
name string
source string
path string
}{
{
name: "child null",
source: `version: 6
pipelines:
parent: {input: {module: input}, structured_output_repair_attempts: 2}
child:
extends: parent
structured_output_repair_attempts: null
`,
path: "/pipelines/child/structured_output_repair_attempts",
},
{
name: "inherited null",
source: `version: 6
pipelines:
parent: {input: {module: input}, structured_output_repair_attempts: null}
child: {extends: parent}
`,
path: "/pipelines/parent/structured_output_repair_attempts",
},
} {
t.Run(test.name, func(t *testing.T) {
_, err := loadConfigSource(configkit.Bytes("null.yml", []byte(test.source)), LoadOptions{})
assertDocumentDiagnosticAt(t, err, test.path)
})
}
}
func TestOverlayInheritancePrecedesOverrides(t *testing.T) {
t.Parallel()
@@ -283,12 +247,12 @@ func TestOverlayInheritancePrecedesOverrides(t *testing.T) {
pipelines:
parent:
input: {module: input}
structured_output_repair_attempts: 1
max_additional_turns: 1
inherited:
extends: parent
overridden:
extends: parent
structured_output_repair_attempts: 0
max_additional_turns: 0
cancelled:
extends: parent
input: {module: input}
@@ -297,7 +261,7 @@ pipelines:
}
if err := os.WriteFile(filepath.Join(overlays, "development.yml"), []byte(`pipelines:
parent:
structured_output_repair_attempts: 2
max_additional_turns: 2
cancelled:
extends: null
`), 0o600); err != nil {
@@ -308,32 +272,32 @@ pipelines:
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if got := *loaded.Config.Pipelines["inherited"].StructuredOutputRepairAttempts; got != 2 {
t.Fatalf("overlay-inherited repair budget = %d, want 2", got)
if got := *loaded.Config.Pipelines["inherited"].MaxAdditionalTurns; got != 2 {
t.Fatalf("overlay-inherited turn allowance = %d, want 2", got)
}
if got := *loaded.Config.Pipelines["overridden"].StructuredOutputRepairAttempts; got != 0 {
t.Fatalf("child replacement repair budget = %d, want 0", got)
if got := *loaded.Config.Pipelines["overridden"].MaxAdditionalTurns; got != 0 {
t.Fatalf("child replacement turn allowance = %d, want 0", got)
}
if loaded.Config.Pipelines["cancelled"].StructuredOutputRepairAttempts != nil {
if loaded.Config.Pipelines["cancelled"].MaxAdditionalTurns != nil {
t.Fatalf("cancelled child inherited parent settings: %#v", loaded.Config.Pipelines["cancelled"])
}
withOverride, err := Load(main, LoadOptions{
Overlay: "development",
Overrides: []configkit.Override{{
Path: configkit.Keys("pipelines", "parent", "structured_output_repair_attempts"),
Path: configkit.Keys("pipelines", "parent", "max_additional_turns"),
Value: 3,
Flag: "--repair-budget",
Flag: "--set",
}},
})
if err != nil {
t.Fatalf("Load() with explicit override error = %v", err)
}
if got := *withOverride.Config.Pipelines["inherited"].StructuredOutputRepairAttempts; got != 2 {
t.Fatalf("inherited repair budget after parent override = %d, want 2", got)
if got := *withOverride.Config.Pipelines["inherited"].MaxAdditionalTurns; got != 2 {
t.Fatalf("inherited turn allowance after parent override = %d, want 2", got)
}
if got := *withOverride.Config.Pipelines["parent"].StructuredOutputRepairAttempts; got != 3 {
t.Fatalf("parent override repair budget = %d, want 3", got)
if got := *withOverride.Config.Pipelines["parent"].MaxAdditionalTurns; got != 3 {
t.Fatalf("parent override turn allowance = %d, want 3", got)
}
}
@@ -465,7 +429,7 @@ func TestInheritedInspectionRetainsOriginsRedactionAndOwnership(t *testing.T) {
if err := os.WriteFile(parentFile, []byte(`pipelines:
parent:
input: {module: input}
structured_output_repair_attempts: 2
max_additional_turns: 2
references:
source: {path: ./shared.txt}
artifacts:
@@ -485,7 +449,7 @@ func TestInheritedInspectionRetainsOriginsRedactionAndOwnership(t *testing.T) {
if err := os.WriteFile(childFile, []byte(`pipelines:
child:
extends: parent
structured_output_repair_attempts: 0
max_additional_turns: 0
`), 0o600); err != nil {
t.Fatal(err)
}
@@ -518,8 +482,8 @@ config:
if got := child.References["source"].Path; got != wantPath {
t.Fatalf("runtime inherited path = %q, want %q", got, wantPath)
}
if child.StructuredOutputRepairAttempts == nil || *child.StructuredOutputRepairAttempts != 0 {
t.Fatalf("runtime inherited repair setting = %#v", child.StructuredOutputRepairAttempts)
if child.MaxAdditionalTurns == nil || *child.MaxAdditionalTurns != 0 {
t.Fatalf("runtime inherited turn allowance = %#v", child.MaxAdditionalTurns)
}
var effective bytes.Buffer
@@ -527,7 +491,7 @@ config:
t.Fatalf("WriteEffective() error = %v", err)
}
if !strings.Contains(effective.String(), wantPath) || strings.Contains(effective.String(), "extends:") ||
!strings.Contains(effective.String(), "structured_output_repair_attempts: 0") {
!strings.Contains(effective.String(), "max_additional_turns: 0") {
t.Fatalf("effective configuration is inconsistent with runtime conversion: %s", effective.String())
}
for _, secret := range []string{"fake-parent-secret", "fake-overlay-secret"} {
@@ -540,7 +504,7 @@ config:
name string
path configkit.Path
}{
{name: "repair setting", path: configkit.Keys("pipelines", "child", "structured_output_repair_attempts")},
{name: "turn allowance", path: configkit.Keys("pipelines", "child", "max_additional_turns")},
{name: "normalized reference", path: configkit.Keys("pipelines", "child", "references", "source", "path")},
} {
t.Run(test.name, func(t *testing.T) {

View File

@@ -72,23 +72,11 @@ func documentDefinition(state *documentLoadState) configkit.Definition[Document]
func documentDefaults() Document {
return Document{
Pipelines: map[string]PipelineDocument{},
Concurrency: &ConcurrencyDocument{
TotalLLM: documentValue(defaultLLMConcurrency),
StageWorkers: map[string]int{},
},
Output: &OutputDocument{Directory: documentValue("./notarius-output")},
Cache: &CacheDocument{
ChunkPlans: &ChunkPlanCacheDocument{
Directory: documentValue(""),
Mode: documentValue("auto"),
},
Checkpoints: &CheckpointCacheDocument{
Enabled: documentValue(false),
Directory: documentValue(""),
},
},
Debug: &DebugDocument{Directory: documentValue("./notarius-debug")},
Pipelines: map[string]PipelineDocument{},
Concurrency: &ConcurrencyDocument{Workers: documentValue(defaultWorkers)},
Output: &OutputDocument{Directory: documentValue("./notarius-output")},
CacheDirectory: documentValue(""),
Debug: &DebugDocument{Directory: documentValue("/tmp/notarius-debug")},
}
}

View File

@@ -112,36 +112,14 @@ func newLoadedConfig(result *configkit.Result[Document]) (*LoadedConfig, error)
func documentToConfig(document Document) (Config, error) {
config := Default()
config.Pipelines = documentPipelinesToRuntime(document.Pipelines)
if document.Concurrency != nil {
config.Concurrency.StageWorkers = documentIntMap(document.Concurrency.StageWorkers)
if document.Concurrency.TotalLLM != nil {
config.Concurrency.TotalLLM = *document.Concurrency.TotalLLM
}
if document.Concurrency != nil && document.Concurrency.Workers != nil {
config.Concurrency.Workers = *document.Concurrency.Workers
}
if document.Output != nil && document.Output.Directory != nil {
config.Output.Directory = *document.Output.Directory
}
if document.Cache != nil {
if document.Cache.ChunkPlans != nil {
if document.Cache.ChunkPlans.Directory != nil {
config.Cache.ChunkPlans.Directory = *document.Cache.ChunkPlans.Directory
}
if document.Cache.ChunkPlans.Mode != nil {
mode, err := pipeline.ParseChunkCacheMode(*document.Cache.ChunkPlans.Mode)
if err != nil {
return Config{}, fmt.Errorf("convert chunk-plan cache mode: %w", err)
}
config.Cache.ChunkPlans.Mode = mode
}
}
if document.Cache.Checkpoints != nil {
if document.Cache.Checkpoints.Enabled != nil {
config.Cache.Checkpoints.Enabled = *document.Cache.Checkpoints.Enabled
}
if document.Cache.Checkpoints.Directory != nil {
config.Cache.Checkpoints.Directory = *document.Cache.Checkpoints.Directory
}
}
if document.CacheDirectory != nil {
config.CacheDirectory = *document.CacheDirectory
}
if document.Debug != nil && document.Debug.Directory != nil {
config.Debug.Directory = *document.Debug.Directory
@@ -162,12 +140,13 @@ func documentPipelinesToRuntime(documents map[string]PipelineDocument) map[strin
func documentPipelineToRuntime(id string, document PipelineDocument) pipeline.PipelineProfile {
profile := pipeline.PipelineProfile{
ID: id,
StructuredOutputRepairAttempts: cloneDocumentInt(document.StructuredOutputRepairAttempts),
ValidationPolicy: documentPolicyToRuntime(document.ValidationPolicy),
Input: documentBindingToRuntime(document.Input),
Artifacts: documentLanesToRuntime(document.Artifacts),
References: documentReferencesToRuntime(document.References),
Mode: cloneValue(document.Mode),
MaxAdditionalTurns: cloneValue(document.MaxAdditionalTurns),
Cache: cloneValue(document.Cache),
ID: id,
Input: documentBindingToRuntime(document.Input),
Artifacts: documentLanesToRuntime(document.Artifacts),
References: documentReferencesToRuntime(document.References),
}
if document.LLMProfile != nil {
profile.LLMProfile = *document.LLMProfile
@@ -220,12 +199,12 @@ func documentLanesToRuntime(documents map[string]ArtifactLaneDocument) map[strin
func documentBindingToRuntime(document BindingDocument) pipeline.ModuleBinding {
binding := pipeline.ModuleBinding{
Module: document.Module,
StructuredOutputRepairAttempts: cloneDocumentInt(document.StructuredOutputRepairAttempts),
ValidationPolicy: documentPolicyToRuntime(document.ValidationPolicy),
Retries: document.Retries,
Options: documentOptions(document.Options),
References: documentReferencesToRuntime(document.References),
Mode: cloneValue(document.Mode),
MaxAdditionalTurns: cloneValue(document.MaxAdditionalTurns),
Cache: cloneValue(document.Cache),
Module: document.Module,
Options: documentOptions(document.Options),
References: documentReferencesToRuntime(document.References),
}
if document.LLMProfile != nil {
binding.LLMProfile = *document.LLMProfile
@@ -242,10 +221,8 @@ func documentBindingToRuntime(document BindingDocument) pipeline.ModuleBinding {
func documentValidatorToRuntime(document ValidatorDocument) pipeline.ModuleBinding {
binding := pipeline.ModuleBinding{
Module: document.Module,
StructuredOutputRepairAttempts: cloneDocumentInt(document.StructuredOutputRepairAttempts),
Retries: document.Retries,
Options: documentOptions(document.Options),
Module: document.Module,
Options: documentOptions(document.Options),
}
if document.LLMProfile != nil {
binding.LLMProfile = *document.LLMProfile
@@ -270,45 +247,6 @@ func documentReferencesToRuntime(documents map[string]ReferenceDocument) map[str
return references
}
func documentPolicyToRuntime(document *ValidationPolicyDocument) *pipeline.ValidationPolicyOverride {
if document == nil {
return nil
}
policy := pipeline.ValidationPolicyOverride{}
if document.ProducerStructuralFailure != nil {
value := *document.ProducerStructuralFailure
policy.ProducerStructuralFailure = &value
}
if document.SemanticRejection != nil {
value := *document.SemanticRejection
policy.SemanticRejection = &value
}
if document.ValidatorFailure != nil {
value := *document.ValidatorFailure
policy.ValidatorFailure = &value
}
return &policy
}
func documentIntMap(values map[string]int) map[string]int {
if values == nil {
return nil
}
result := make(map[string]int, len(values))
for key, value := range values {
result[key] = value
}
return result
}
func cloneDocumentInt(value *int) *int {
if value == nil {
return nil
}
result := *value
return &result
}
func documentOptions(values map[string]any) map[string]any {
if values == nil {
return nil

View File

@@ -49,7 +49,7 @@ func TestLoadedConfigConvertsDocumentsAndResolvesRuntimePipeline(t *testing.T) {
pipelines:
session:
llm_profile: local
structured_output_repair_attempts: 0
max_additional_turns: 0
input:
module: input
chunk:
@@ -69,7 +69,7 @@ pipelines:
t.Fatalf("loadConfigSource() error = %v", err)
}
profile := loaded.Config.Pipelines["session"]
if profile.LLMProfile != "local" || profile.StructuredOutputRepairAttempts == nil || *profile.StructuredOutputRepairAttempts != 0 {
if profile.LLMProfile != "local" || profile.MaxAdditionalTurns == nil || *profile.MaxAdditionalTurns != 0 {
t.Fatalf("pipeline profile = %#v", profile)
}
if !profile.Chunk.Validators.Set || profile.Chunk.Validators.Validators == nil || len(profile.Chunk.Validators.Validators) != 0 {
@@ -177,8 +177,8 @@ pipelines:
}
func TestOverrideCannotRepairTypedDecodeFailure(t *testing.T) {
_, err := loadConfigSource(configkit.Bytes("invalid.yml", []byte("version: 6\nconcurrency: {total_llm: invalid}\n")), LoadOptions{
Overrides: []configkit.Override{{Path: configkit.Keys("concurrency", "total_llm"), Value: 2, Flag: "--total-llm"}},
_, err := loadConfigSource(configkit.Bytes("invalid.yml", []byte("version: 6\nconcurrency: {workers: invalid}\n")), LoadOptions{
Overrides: []configkit.Override{{Path: configkit.Keys("concurrency", "workers"), Value: 2, Flag: "--workers"}},
})
if err == nil {
t.Fatal("override unexpectedly repaired invalid typed input")

View File

@@ -2,10 +2,9 @@ package config
import (
"fmt"
"sort"
"os"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
configkit "go.maximumdirect.net/configkit"
)
@@ -15,13 +14,15 @@ func resolveDocumentOperations(document *Document, resolution *configkit.Resolut
}
normalizeDocumentState(document, resolution)
normalizeDocumentPaths(document, resolution, state)
normalizeDocumentPipelines(document, resolution, state)
if document.Concurrency != nil {
document.Concurrency.StageWorkers = normalizeDocumentMap(document.Concurrency.StageWorkers,
configkit.Keys("concurrency", "stage_workers"), "stage worker identifier",
documentNormalizer{Resolution: resolution}, state)
if document.CacheDirectory != nil && *document.CacheDirectory == "" {
root, err := DefaultCacheRoot(os.UserCacheDir)
if err != nil {
return err
}
// Empty is an explicit selection of the per-user default, not a physical root.
configkit.Normalize(resolution, configkit.Keys("cache_directory"), &document.CacheDirectory, documentValue(root), "selected per-user cache directory")
}
deriveDocumentExtractWorkers(document, resolution, state)
normalizeDocumentPipelines(document, resolution, state)
return nil
}
@@ -29,15 +30,7 @@ func normalizeDocumentState(document *Document, resolution *configkit.Resolution
if document.Output != nil {
normalizeDocumentString(resolution, configkit.Keys("output", "directory"), &document.Output.Directory, "trimmed output directory")
}
if document.Cache != nil {
if document.Cache.ChunkPlans != nil {
normalizeDocumentString(resolution, configkit.Keys("cache", "chunk_plans", "directory"), &document.Cache.ChunkPlans.Directory, "trimmed chunk-plan cache directory")
normalizeDocumentString(resolution, configkit.Keys("cache", "chunk_plans", "mode"), &document.Cache.ChunkPlans.Mode, "trimmed chunk-plan cache mode")
}
if document.Cache.Checkpoints != nil {
normalizeDocumentString(resolution, configkit.Keys("cache", "checkpoints", "directory"), &document.Cache.Checkpoints.Directory, "trimmed checkpoint cache directory")
}
}
normalizeDocumentString(resolution, configkit.Keys("cache_directory"), &document.CacheDirectory, "trimmed cache directory")
if document.Debug != nil {
normalizeDocumentString(resolution, configkit.Keys("debug", "directory"), &document.Debug.Directory, "trimmed debug directory")
}
@@ -50,120 +43,36 @@ func normalizeDocumentString(resolution *configkit.Resolution, path configkit.Pa
configkit.Normalize(resolution, path, target, documentValue(strings.TrimSpace(**target)), reason)
}
func deriveDocumentExtractWorkers(document *Document, resolution *configkit.Resolution, state *documentLoadState) {
if document.Concurrency == nil || document.Concurrency.TotalLLM == nil {
return
}
if document.Concurrency.StageWorkers == nil {
document.Concurrency.StageWorkers = make(map[string]int)
}
path := configkit.Keys("concurrency", "stage_workers", "extract")
// Presence belongs to the input spelling, not the key produced by trimming.
// A moved explicit count must never be replaced by a derived default.
if resolution.IsExplicit(state.originalPath(path)) {
return
}
workers := document.Concurrency.StageWorkers
value := workers["extract"]
configkit.Default(resolution, path, &value, *document.Concurrency.TotalLLM,
"derived extraction workers from total LLM concurrency", configkit.Keys("concurrency", "total_llm"))
workers["extract"] = value
}
func validateDocumentOperations(document *Document, validation *configkit.Validation) {
if document == nil {
return
}
validateDocumentConcurrency(document.Concurrency, validation)
validateDocumentState(document, validation)
validateDocumentPipelines(document, validation)
}
func validateDocumentConcurrency(concurrency *ConcurrencyDocument, validation *configkit.Validation) {
if concurrency == nil {
validation.Error(configkit.Keys("concurrency"), "must not be null")
return
if document.Concurrency == nil || document.Concurrency.Workers == nil {
validation.Error(configkit.Keys("concurrency", "workers"), "must be a positive integer")
} else if err := validateWorkers(*document.Concurrency.Workers); err != nil {
validation.Error(configkit.Keys("concurrency", "workers"), err.Error())
}
totalPath := configkit.Keys("concurrency", "total_llm")
if concurrency.TotalLLM == nil {
validation.Error(totalPath, "must not be null")
return
}
totalValid := true
if err := validateTotalLLMConcurrency(*concurrency.TotalLLM); err != nil {
validation.Error(totalPath, err.Error())
totalValid = false
}
keys := make([]string, 0, len(concurrency.StageWorkers))
for key := range concurrency.StageWorkers {
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
if err := validateStageWorkerKey(key); err != nil {
validation.Error(configkit.Keys("concurrency", "stage_workers").Key(key), err.Error())
}
}
if !totalValid {
return
}
if workers, ok := concurrency.StageWorkers["extract"]; ok {
if err := validateExtractWorkerCount(workers, *concurrency.TotalLLM); err != nil {
validation.Error(configkit.Keys("concurrency", "stage_workers", "extract"), err.Error())
}
}
}
func validateDocumentState(document *Document, validation *configkit.Validation) {
if document.Output == nil || document.Output.Directory == nil {
validation.Error(configkit.Keys("output", "directory"), "must not be empty")
} else if err := validateRequiredPath("output.directory", *document.Output.Directory); err != nil {
validation.Error(configkit.Keys("output", "directory"), err.Error())
}
if document.Cache == nil || document.Cache.ChunkPlans == nil || document.Cache.ChunkPlans.Mode == nil {
validation.Error(configkit.Keys("cache", "chunk_plans", "mode"), "must be a supported cache mode")
} else if err := validateChunkPlanCacheMode(*document.Cache.ChunkPlans.Mode); err != nil {
validation.Error(configkit.Keys("cache", "chunk_plans", "mode"), err.Error())
if document.CacheDirectory == nil {
validation.Error(configkit.Keys("cache_directory"), "must not be null")
} else if err := validateOptionalPath("cache_directory", *document.CacheDirectory); err != nil {
validation.Error(configkit.Keys("cache_directory"), err.Error())
}
if document.Cache != nil && document.Cache.ChunkPlans != nil && document.Cache.ChunkPlans.Directory != nil {
if err := validateOptionalPath("cache.chunk_plans.directory", *document.Cache.ChunkPlans.Directory); err != nil {
validation.Error(configkit.Keys("cache", "chunk_plans", "directory"), err.Error())
}
}
if document.Cache != nil && document.Cache.Checkpoints != nil && document.Cache.Checkpoints.Directory != nil {
if err := validateOptionalPath("cache.checkpoints.directory", *document.Cache.Checkpoints.Directory); err != nil {
validation.Error(configkit.Keys("cache", "checkpoints", "directory"), err.Error())
}
}
if document.Debug == nil || document.Debug.Directory == nil {
validation.Error(configkit.Keys("debug", "directory"), "must not be empty")
} else if err := validateRequiredPath("debug.directory", *document.Debug.Directory); err != nil {
validation.Error(configkit.Keys("debug", "directory"), "must not be null")
} else if err := validateOptionalPath("debug.directory", *document.Debug.Directory); err != nil {
validation.Error(configkit.Keys("debug", "directory"), err.Error())
}
validateDocumentPipelines(document, validation)
}
func validateTotalLLMConcurrency(total int) error {
if total <= 0 {
return fmt.Errorf("total LLM concurrency must be greater than zero")
}
return nil
}
func validateStageWorkerKey(key string) error {
if strings.TrimSpace(key) == "" {
return fmt.Errorf("concurrency.stage_workers key must not be empty")
}
if key != "extract" {
return fmt.Errorf("concurrency.stage_workers key %q is not supported", key)
}
return nil
}
func validateExtractWorkerCount(workers, total int) error {
if workers < 1 || workers > total {
return fmt.Errorf("concurrency.stage_workers.extract must be between 1 and concurrency.total_llm (%d)", total)
func validateWorkers(workers int) error {
if workers <= 0 {
return fmt.Errorf("concurrency.workers must be greater than zero")
}
return nil
}
@@ -174,17 +83,9 @@ func validateRequiredPath(name, value string) error {
}
return validateOptionalPath(name, value)
}
func validateOptionalPath(name, value string) error {
if strings.ContainsRune(value, '\x00') {
return fmt.Errorf("%s must not contain NUL", name)
}
return nil
}
func validateChunkPlanCacheMode(value string) error {
if _, err := pipeline.ParseChunkCacheMode(value); err != nil {
return fmt.Errorf("cache.chunk_plans.mode: %w", err)
}
return nil
}

View File

@@ -5,203 +5,109 @@ import (
"errors"
"os"
"path/filepath"
"strings"
"testing"
configkit "go.maximumdirect.net/configkit"
)
func TestWorkerKeyNormalizationPreservesExplicitCount(t *testing.T) {
func TestDocumentWorkerCount(t *testing.T) {
t.Parallel()
for _, test := range []struct {
name, workers string
want int
errorPath string
name, body string
want int
invalid bool
}{
{name: "omitted", workers: "{}", want: 4},
{name: "padded", workers: "{' extract ': 2}", want: 2},
{name: "explicit invalid", workers: "{' extract ': 0}", errorPath: "/concurrency/stage_workers/extract"},
{name: "collision", workers: "{' extract ': 2, extract: 3}", errorPath: "/concurrency/stage_workers/extract"},
{"omitted", "", 64, false},
{"serial", "concurrency: {workers: 1}", 1, false},
{"large", "concurrency: {workers: 1000000}", 1000000, false},
{"zero", "concurrency: {workers: 0}", 0, true},
{"negative", "concurrency: {workers: -1}", 0, true},
} {
t.Run(test.name, func(t *testing.T) {
loaded, err := loadConfigSource(configkit.Bytes("workers.yml", []byte(
"version: 6\nconcurrency:\n total_llm: 4\n stage_workers: "+test.workers+"\n")), LoadOptions{})
if test.errorPath != "" {
assertDocumentDiagnosticAt(t, err, test.errorPath)
loaded, err := loadConfigSource(configkit.Bytes("workers.yml", []byte("version: 6\n"+test.body+"\n")), LoadOptions{})
if test.invalid {
assertDocumentDiagnosticAt(t, err, "/concurrency/workers")
return
}
if err != nil {
t.Fatal(err)
}
if got := loaded.Config.Concurrency.StageWorkers["extract"]; got != test.want {
t.Fatalf("workers = %d, want %d", got, test.want)
}
var explanation bytes.Buffer
if err := loaded.WriteExplanation(&explanation, "/concurrency/stage_workers"); err != nil {
t.Fatal(err)
}
if test.name == "padded" && (!strings.Contains(explanation.String(), "normalization") ||
!strings.Contains(explanation.String(), " extract ") ||
strings.Contains(explanation.String(), "derived extraction workers")) {
t.Fatalf("explicit worker provenance = %s", explanation.String())
if loaded.Config.Concurrency.Workers != test.want {
t.Fatalf("workers = %d", loaded.Config.Concurrency.Workers)
}
})
}
}
func TestDocumentLoadAppliesOperationalDefaultsAndNormalization(t *testing.T) {
func TestDocumentStateDirectorySelection(t *testing.T) {
t.Parallel()
loaded, err := loadDocumentSource(configkit.Bytes("document.yml", []byte(`version: 6
concurrency:
total_llm: 4
output:
directory: " ./output "
cache:
chunk_plans:
mode: " refresh "
directory: " "
checkpoints:
directory: " "
debug:
directory: " ./debug "
`)), LoadOptions{})
userRoot, err := os.UserCacheDir()
if err != nil {
t.Fatal(err)
}
if loaded.Config.Concurrency.TotalLLM == nil || *loaded.Config.Concurrency.TotalLLM != 4 || loaded.Config.Concurrency.StageWorkers["extract"] != 4 {
t.Fatalf("concurrency = %#v", loaded.Config.Concurrency)
}
workingDirectory, err := os.Getwd()
if err != nil {
t.Fatal(err)
}
if loaded.Config.Output.Directory == nil || *loaded.Config.Output.Directory != filepath.Join(workingDirectory, "output") ||
loaded.Config.Cache.ChunkPlans.Mode == nil || *loaded.Config.Cache.ChunkPlans.Mode != "refresh" ||
loaded.Config.Cache.ChunkPlans.Directory == nil || *loaded.Config.Cache.ChunkPlans.Directory != "" ||
loaded.Config.Cache.Checkpoints.Directory == nil || *loaded.Config.Cache.Checkpoints.Directory != "" ||
loaded.Config.Debug.Directory == nil || *loaded.Config.Debug.Directory != filepath.Join(workingDirectory, "debug") {
t.Fatalf("state = %#v, %#v, %#v", loaded.Config.Output, loaded.Config.Cache, loaded.Config.Debug)
}
if diagnostics := loaded.Diagnostics(); len(diagnostics) != 0 {
t.Fatalf("diagnostics = %#v", diagnostics)
}
extractHistory := loaded.Metadata().History(configkit.Keys("concurrency", "stage_workers", "extract"))
if len(extractHistory) == 0 || extractHistory[len(extractHistory)-1].Origin.Operation != configkit.ChangeDerivedDefault {
t.Fatalf("extract history = %#v", extractHistory)
}
outputHistory := loaded.Metadata().History(configkit.Keys("output", "directory"))
if len(outputHistory) == 0 || outputHistory[len(outputHistory)-1].Origin.Operation != configkit.ChangeNormalization {
t.Fatalf("output history = %#v", outputHistory)
for _, test := range []struct{ name, body, cache, debug string }{
{"omitted", "", filepath.Join(userRoot, "notarius"), "/tmp/notarius-debug"},
{"empty", "cache_directory: ''\ndebug: {directory: ''}", filepath.Join(userRoot, "notarius"), ""},
{"explicit", "cache_directory: /var/cache/fixture\ndebug: {directory: /tmp/fixture-debug}", "/var/cache/fixture", "/tmp/fixture-debug"},
} {
t.Run(test.name, func(t *testing.T) {
loaded, err := loadConfigSource(configkit.Bytes("directories.yml", []byte("version: 6\n"+test.body+"\n")), LoadOptions{})
if err != nil {
t.Fatal(err)
}
if loaded.Config.CacheDirectory != test.cache || loaded.Config.Debug.Directory != test.debug {
t.Fatalf("directories = %q, %q", loaded.Config.CacheDirectory, loaded.Config.Debug.Directory)
}
var effective bytes.Buffer
if err := loaded.WriteEffective(&effective); err != nil {
t.Fatal(err)
}
})
}
}
func TestDocumentLoadDerivesWorkersAfterOverlayAndCLIOverrides(t *testing.T) {
func TestDirectoryPathsFollowImportedSource(t *testing.T) {
t.Parallel()
directory := t.TempDir()
main := filepath.Join(directory, "config.yml")
overlays := filepath.Join(directory, "overlays")
if err := os.Mkdir(overlays, 0o755); err != nil {
dir := t.TempDir()
fragmentDir := filepath.Join(dir, "fragments")
if err := os.Mkdir(fragmentDir, 0755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(main, []byte("version: 6\nconcurrency:\n total_llm: 4\n"), 0o600); err != nil {
main := filepath.Join(dir, "config.yml")
if err := os.WriteFile(main, []byte("version: 6\nconfig:\n imports:\n - path: fragments/state.yml\n required: true\n"), 0600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(overlays, "development.yml"), []byte("concurrency:\n total_llm: 3\n"), 0o600); err != nil {
if err := os.WriteFile(filepath.Join(fragmentDir, "state.yml"), []byte("cache_directory: selected\ndebug: {directory: evidence}\noutput: {directory: deliverables}\n"), 0600); err != nil {
t.Fatal(err)
}
overlaid, err := loadDocument(main, LoadOptions{Overlay: "development"})
loaded, err := Load(main, LoadOptions{})
if err != nil {
t.Fatal(err)
}
if overlaid.Config.Concurrency.TotalLLM == nil || *overlaid.Config.Concurrency.TotalLLM != 3 || overlaid.Config.Concurrency.StageWorkers["extract"] != 3 {
t.Fatalf("overlaid concurrency = %#v", overlaid.Config.Concurrency)
}
overridden, err := loadDocument(main, LoadOptions{Overrides: []configkit.Override{{
Path: configkit.Keys("concurrency", "total_llm"), Value: 2, Flag: "--total-llm",
}}})
if err != nil {
t.Fatal(err)
}
if overridden.Config.Concurrency.TotalLLM == nil || *overridden.Config.Concurrency.TotalLLM != 2 || overridden.Config.Concurrency.StageWorkers["extract"] != 2 {
t.Fatalf("overridden concurrency = %#v", overridden.Config.Concurrency)
}
overlaidAndOverridden, err := loadDocument(main, LoadOptions{Overlay: "development", Overrides: []configkit.Override{{
Path: configkit.Keys("concurrency", "total_llm"), Value: 2, Flag: "--total-llm",
}}})
if err != nil {
t.Fatal(err)
}
if overlaidAndOverridden.Config.Concurrency.TotalLLM == nil || *overlaidAndOverridden.Config.Concurrency.TotalLLM != 2 || overlaidAndOverridden.Config.Concurrency.StageWorkers["extract"] != 2 {
t.Fatalf("overlaid and overridden concurrency = %#v", overlaidAndOverridden.Config.Concurrency)
if loaded.Config.CacheDirectory != filepath.Join(fragmentDir, "selected") || loaded.Config.Debug.Directory != filepath.Join(fragmentDir, "evidence") || loaded.Config.Output.Directory != filepath.Join(fragmentDir, "deliverables") {
t.Fatalf("source-relative directories lost: %#v", loaded.Config)
}
}
func TestDocumentLoadRetainsExplicitWorkerAndRejectsInvalidOperationalValues(t *testing.T) {
func TestDocumentRejectsRetiredControls(t *testing.T) {
t.Parallel()
loaded, err := loadDocumentSource(configkit.Bytes("explicit.yml", []byte(`version: 6
concurrency:
total_llm: 4
stage_workers:
extract: 0
`)), LoadOptions{})
if loaded != nil {
t.Fatalf("loaded = %#v, want validation failure", loaded.Config)
}
assertDocumentDiagnostic(t, err, configkit.CodeValidateError, "/concurrency/stage_workers/extract")
_, err = loadDocumentSource(configkit.Bytes("invalid.yml", []byte(`version: 6
concurrency:
total_llm: 0
output:
directory: ""
`)), LoadOptions{})
var loadError *configkit.LoadError
if !errors.As(err, &loadError) {
t.Fatalf("error = %v, want ConfigKit load error", err)
}
seen := make(map[string]bool)
for _, diagnostic := range loadError.Diagnostics() {
if diagnostic.Code != configkit.CodeValidateError || diagnostic.Primary == nil {
t.Fatalf("diagnostic = %#v", diagnostic)
}
seen[diagnostic.Path.String()] = true
}
for _, path := range []string{"/concurrency/total_llm", "/output/directory"} {
if !seen[path] {
t.Fatalf("diagnostics missing %s: %#v", path, loadError.Diagnostics())
for _, body := range []string{
"concurrency: {total_llm: 4}",
"concurrency: {stage_workers: {extract: 4}}",
"cache: {chunk_plans: {mode: auto}}",
"pipelines: {main: {input: {module: input}, retries: 1}}",
"pipelines: {main: {input: {module: input}, structured_output_repair_attempts: 0}}",
"pipelines: {main: {input: {module: input}, validation_policy: {validator_failure: fail_run}}}",
"pipelines: {main: {input: {module: input}, chunk: {module: chunk, retries: 1}}}",
"pipelines: {main: {input: {module: input}, chunk: {module: chunk, validators: [{module: validate, max_additional_turns: 1}]}}}",
} {
_, err := loadConfigSource(configkit.Bytes("retired.yml", []byte("version: 6\n"+body+"\n")), LoadOptions{})
if err == nil {
t.Fatalf("accepted retired controls: %s", body)
}
}
}
func TestDocumentLoadDistinguishesRequiredAndOptionalPathValues(t *testing.T) {
t.Parallel()
loaded, err := loadDocumentSource(configkit.Bytes("cache.yml", []byte(`version: 6
cache:
chunk_plans:
directory: " "
checkpoints:
directory: " "
`)), LoadOptions{})
if err != nil {
t.Fatal(err)
}
if loaded.Config.Cache.ChunkPlans.Directory == nil || *loaded.Config.Cache.ChunkPlans.Directory != "" ||
loaded.Config.Cache.Checkpoints.Directory == nil || *loaded.Config.Cache.Checkpoints.Directory != "" {
t.Fatalf("cache directories = %#v", loaded.Config.Cache)
}
_, err = loadDocumentSource(configkit.Bytes("output.yml", []byte("version: 6\noutput:\n directory: ' '\n")), LoadOptions{})
assertDocumentDiagnostic(t, err, configkit.CodeValidateError, "/output/directory")
}
func assertDocumentDiagnostic(t *testing.T, err error, code configkit.Code, path string) {
t.Helper()
var loadError *configkit.LoadError

View File

@@ -83,10 +83,8 @@ func validateDocumentPipelinePresence(document *Document, resolution *configkit.
}
func validatePipelineDocumentPresence(profile PipelineDocument, path configkit.Path, resolution *configkit.Resolution) {
validateDocumentControls(profile.Mode, profile.MaxAdditionalTurns, profile.Cache, path, validation)
rejectDocumentNull(resolution, path.Key("llm_profile"), profile.LLMProfile == nil)
rejectDocumentNull(resolution, path.Key("structured_output_repair_attempts"), profile.StructuredOutputRepairAttempts == nil)
rejectDocumentNull(resolution, path.Key("validation_policy"), profile.ValidationPolicy == nil)
validateDocumentPolicyPresence(profile.ValidationPolicy, path.Key("validation_policy"), resolution)
stepsPath := path.Key("steps")
if resolution.IsExplicit(stepsPath) && profile.Steps == nil {
@@ -136,9 +134,6 @@ func validateOptionalBindingDocumentPresence(binding *BindingDocument, path conf
func validateBindingDocumentPresence(binding BindingDocument, path configkit.Path, resolution *configkit.Resolution) {
rejectDocumentNull(resolution, path.Key("llm_profile"), binding.LLMProfile == nil)
rejectDocumentNull(resolution, path.Key("structured_output_repair_attempts"), binding.StructuredOutputRepairAttempts == nil)
rejectDocumentNull(resolution, path.Key("validation_policy"), binding.ValidationPolicy == nil)
validateDocumentPolicyPresence(binding.ValidationPolicy, path.Key("validation_policy"), resolution)
if binding.Validators == nil && resolution.IsExplicit(path.Key("validators")) {
resolution.Error(path.Key("validators"), "must not be null")
}
@@ -146,7 +141,6 @@ func validateBindingDocumentPresence(binding BindingDocument, path configkit.Pat
for index, validator := range binding.Validators {
validatorPath := path.Key("validators").Index(index)
rejectDocumentNull(resolution, validatorPath.Key("llm_profile"), validator.LLMProfile == nil)
rejectDocumentNull(resolution, validatorPath.Key("structured_output_repair_attempts"), validator.StructuredOutputRepairAttempts == nil)
}
}
@@ -165,16 +159,8 @@ func rejectDocumentNull(resolution *configkit.Resolution, path configkit.Path, i
}
}
func validateDocumentPolicyPresence(policy *ValidationPolicyDocument, path configkit.Path, resolution *configkit.Resolution) {
if policy == nil {
return
}
rejectDocumentNull(resolution, path.Key("producer_structural_failure"), policy.ProducerStructuralFailure == nil)
rejectDocumentNull(resolution, path.Key("semantic_rejection"), policy.SemanticRejection == nil)
rejectDocumentNull(resolution, path.Key("validator_failure"), policy.ValidatorFailure == nil)
}
func normalizePipelineDocument(profile *PipelineDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) {
normalizeDocumentControls(&profile.Mode, &profile.Cache, path, resolution)
normalizePipelineDocumentString(resolution, path.Key("llm_profile"), &profile.LLMProfile, "trimmed pipeline LLM profile")
normalizeDocumentBinding(&profile.Input, path.Key("input"), resolution, state)
if profile.Chunk != nil {
@@ -220,6 +206,7 @@ func normalizeArtifactLaneDocument(lane *ArtifactLaneDocument, path configkit.Pa
}
func normalizeDocumentBinding(binding *BindingDocument, path configkit.Path, resolution documentNormalizer, state *documentLoadState) {
normalizeDocumentControls(&binding.Mode, &binding.Cache, path, resolution)
normalizeDocumentValue(resolution, path.Key("module"), &binding.Module, strings.TrimSpace(binding.Module), "trimmed module identifier")
normalizePipelineDocumentString(resolution, path.Key("llm_profile"), &binding.LLMProfile, "trimmed binding LLM profile")
binding.References = normalizeDocumentReferences(binding.References, path.Key("references"), resolution, state)
@@ -329,8 +316,6 @@ func validateDocumentPipelines(document *Document, validation *configkit.Validat
func validatePipelineDocument(profile PipelineDocument, path configkit.Path, validation *configkit.Validation) {
validateSelectedDocumentProfile(profile.LLMProfile, path.Key("llm_profile"), validation)
validateDocumentRepairAttempts(profile.StructuredOutputRepairAttempts, path.Key("structured_output_repair_attempts"), validation)
validateDocumentPolicy(profile.ValidationPolicy, path.Key("validation_policy"), validation)
validateDocumentBinding(profile.Input, path.Key("input"), "input", false, false, validation)
if profile.Chunk != nil {
validateDocumentBinding(*profile.Chunk, path.Key("chunk"), "chunk", true, true, validation)
@@ -383,20 +368,12 @@ func validateDocumentLanes(lanes map[string]ArtifactLaneDocument, path configkit
}
func validateDocumentBinding(binding BindingDocument, path configkit.Path, slot string, referencesAllowed, validatorsAllowed bool, validation *configkit.Validation) {
validateDocumentControls(binding.Mode, binding.MaxAdditionalTurns, binding.Cache, path, validation)
if strings.TrimSpace(binding.Module) == "" {
validation.Error(path.Key("module"), "must not be empty")
}
validateSelectedDocumentProfile(binding.LLMProfile, path.Key("llm_profile"), validation)
validateDocumentRepairAttempts(binding.StructuredOutputRepairAttempts, path.Key("structured_output_repair_attempts"), validation)
if binding.Retries < 0 {
validation.Error(path.Key("retries"), "must be greater than or equal to zero")
}
if binding.ValidationPolicy != nil {
if slot != "chunk" && slot != "extract" && slot != "merge" && slot != "normalize" {
validation.Error(path.Key("validation_policy"), "is not supported for this binding")
}
validateDocumentPolicy(binding.ValidationPolicy, path.Key("validation_policy"), validation)
}
if len(binding.References) > 0 && !referencesAllowed {
validation.Error(path.Key("references"), "are not supported for this binding")
}
@@ -416,10 +393,7 @@ func validateValidatorDocument(validator ValidatorDocument, path configkit.Path,
validation.Error(path.Key("module"), "must not be empty")
}
validateSelectedDocumentProfile(validator.LLMProfile, path.Key("llm_profile"), validation)
validateDocumentRepairAttempts(validator.StructuredOutputRepairAttempts, path.Key("structured_output_repair_attempts"), validation)
if validator.Retries < 0 {
validation.Error(path.Key("retries"), "must be greater than or equal to zero")
}
}
func validateDocumentReferences(references map[string]ReferenceDocument, path configkit.Path, generatedAllowed bool, validation *configkit.Validation) {
@@ -457,39 +431,6 @@ func validateSelectedDocumentProfile(value *string, path configkit.Path, validat
}
}
func validateDocumentRepairAttempts(value *int, path configkit.Path, validation *configkit.Validation) {
if value != nil && (*value < 0 || *value > 3) {
validation.Error(path, "must be between zero and three")
}
}
func validateDocumentPolicy(policy *ValidationPolicyDocument, path configkit.Path, validation *configkit.Validation) {
if policy == nil {
return
}
if policy.ProducerStructuralFailure != nil {
switch *policy.ProducerStructuralFailure {
case pipeline.ProducerStructuralFailureFailRun, pipeline.ProducerStructuralFailureRejectOutput:
default:
validation.Error(path.Key("producer_structural_failure"), "must be fail_run or reject_output")
}
}
if policy.SemanticRejection != nil {
switch *policy.SemanticRejection {
case pipeline.SemanticRejectionFailRun, pipeline.SemanticRejectionRejectOutput:
default:
validation.Error(path.Key("semantic_rejection"), "must be fail_run or reject_output")
}
}
if policy.ValidatorFailure != nil {
switch *policy.ValidatorFailure {
case pipeline.ValidatorFailureWarnContinue, pipeline.ValidatorFailureFailRun:
default:
validation.Error(path.Key("validator_failure"), "must be warn_continue or fail_run")
}
}
}
func sortedPipelineDocumentKeys(values map[string]PipelineDocument) []string {
return sortedDocumentKeys(values)
}
@@ -508,3 +449,32 @@ func sortedDocumentKeys[T any](values map[string]T) []string {
sort.Strings(keys)
return keys
}
func normalizeDocumentControls(mode **pipeline.ExecutionMode, cache **pipeline.CacheMode, path configkit.Path, resolution documentNormalizer) {
if *mode != nil {
value := pipeline.ExecutionMode(strings.TrimSpace(string(**mode)))
if **mode != value {
normalizeDocumentValue(resolution, path.Key("mode"), mode, &value, "trimmed execution mode")
}
}
if *cache != nil {
value := pipeline.CacheMode(strings.TrimSpace(string(**cache)))
if **cache != value {
normalizeDocumentValue(resolution, path.Key("cache"), cache, &value, "trimmed cache mode")
}
}
}
func validateDocumentControls(mode *pipeline.ExecutionMode, turns *int, cache *pipeline.CacheMode, path configkit.Path, validation *configkit.Validation) {
if mode != nil {
if _, err := pipeline.ParseExecutionMode(string(*mode)); err != nil {
validation.Error(path.Key("mode"), err.Error())
}
}
if cache != nil {
if _, err := pipeline.ParseCacheMode(string(*cache)); err != nil {
validation.Error(path.Key("cache"), err.Error())
}
}
if turns != nil && *turns < 0 {
validation.Error(path.Key("max_additional_turns"), "must not be negative")
}
}

View File

@@ -86,7 +86,7 @@ pipelines:
" spells ":
extract:
module: " dnd/spells "
structured_output_repair_attempts: 0
max_additional_turns: 0
validators: []
merge:
module: dnd/merge
@@ -108,7 +108,7 @@ pipelines:
t.Fatalf("normalized pipeline = %#v", loaded.Config.Pipelines)
}
lane, ok := profile.Artifacts["spells"]
if !ok || lane.Extract.StructuredOutputRepairAttempts == nil || *lane.Extract.StructuredOutputRepairAttempts != 0 || lane.Extract.Validators == nil || len(lane.Extract.Validators) != 0 {
if !ok || lane.Extract.MaxAdditionalTurns == nil || *lane.Extract.MaxAdditionalTurns != 0 || lane.Extract.Validators == nil || len(lane.Extract.Validators) != 0 {
t.Fatalf("normalized lane = %#v", profile.Artifacts)
}
selector := lane.Merge.References["previous"].Artifact
@@ -231,13 +231,13 @@ pipelines:
path: "/pipelines/session/chunk/validators",
},
{
name: "negative retry budget",
name: "negative turn allowance",
source: `version: 6
pipelines:
session:
input: {module: seriatim, retries: -1}
input: {module: seriatim, max_additional_turns: -1}
`,
path: "/pipelines/session/input/retries",
path: "/pipelines/session/input/max_additional_turns",
},
}
for _, test := range tests {

View File

@@ -14,7 +14,6 @@ func (e EffectiveConfig) RedactedResolvedPipelinePayload() pipeline.ResolvedPipe
func cloneResolvedPipeline(in pipeline.ResolvedPipeline) pipeline.ResolvedPipeline {
out := in
out.ConfiguredValidationPolicy = cloneValidationPolicyOverride(in.ConfiguredValidationPolicy)
out.Input = redactBinding(cloneModuleBinding(in.Input))
out.Chunk = redactBinding(cloneModuleBinding(in.Chunk))
out.ChunkReferences = pipeline.CloneReferenceTarget(in.ChunkReferences)

View File

@@ -2,7 +2,6 @@ package config
import (
"fmt"
"sort"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
@@ -12,50 +11,20 @@ func (c Config) Validate() error {
if err := validateStateSurfaces(c); err != nil {
return err
}
if err := validateTotalLLMConcurrency(c.Concurrency.TotalLLM); err != nil {
return err
}
if err := validateStageWorkers(c.Concurrency); err != nil {
if err := validateWorkers(c.Concurrency.Workers); err != nil {
return err
}
return validatePipelineProfiles(c.Pipelines)
}
func validateStageWorkers(cfg ConcurrencyConfig) error {
keys := make([]string, 0, len(cfg.StageWorkers))
for key := range cfg.StageWorkers {
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
if err := validateStageWorkerKey(key); err != nil {
return err
}
}
extractWorkers, ok := cfg.StageWorkers["extract"]
if !ok {
extractWorkers = cfg.TotalLLM
}
return validateExtractWorkerCount(extractWorkers, cfg.TotalLLM)
}
func validateStateSurfaces(cfg Config) error {
if err := validateRequiredPath("output.directory", cfg.Output.Directory); err != nil {
return err
}
if err := validateRequiredPath("debug.directory", cfg.Debug.Directory); err != nil {
if err := validateOptionalPath("debug.directory", cfg.Debug.Directory); err != nil {
return err
}
if err := validateChunkPlanCacheMode(string(cfg.Cache.ChunkPlans.Mode)); err != nil {
return err
}
if err := validateOptionalPath("cache.chunk_plans.directory", cfg.Cache.ChunkPlans.Directory); err != nil {
return err
}
if err := validateOptionalPath("cache.checkpoints.directory", cfg.Cache.Checkpoints.Directory); err != nil {
return err
}
return nil
return validateOptionalPath("cache_directory", cfg.CacheDirectory)
}
func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile) error {
@@ -73,13 +42,8 @@ func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile) erro
if profile.ID != "" && strings.TrimSpace(profile.ID) != id {
return fmt.Errorf("pipeline %q profile id %q does not match map key", id, profile.ID)
}
if err := validateStructuredOutputRepairAttempts(fmt.Sprintf("pipeline %q", id), profile.StructuredOutputRepairAttempts); err != nil {
return err
}
if profile.ValidationPolicy != nil {
if err := profile.ValidationPolicy.Validate(); err != nil {
return fmt.Errorf("pipeline %q validation_policy: %w", id, err)
}
if _, err := pipeline.ResolveProducerControls(pipeline.ModuleBinding{}, profile, nil); err != nil {
return fmt.Errorf("pipeline %q: %w", id, err)
}
if err := validateBinding(id, "", "input", profile.Input, false); err != nil {
return err
@@ -160,28 +124,12 @@ func validateBinding(
binding pipeline.ModuleBinding,
referencesAllowed bool,
) error {
if binding.ValidationPolicy != nil {
switch slot {
case "chunk", "extract", "merge", "normalize":
default:
return fmt.Errorf("%s validation_policy is not supported", referenceContext(pipelineID, laneID, slot))
}
if err := binding.ValidationPolicy.Validate(); err != nil {
return fmt.Errorf("%s validation_policy: %w", referenceContext(pipelineID, laneID, slot), err)
}
}
if err := validateStructuredOutputRepairAttempts(referenceContext(pipelineID, laneID, slot), binding.StructuredOutputRepairAttempts); err != nil {
return err
if _, err := pipeline.ResolveProducerControls(binding, pipeline.PipelineProfile{}, nil); err != nil {
return fmt.Errorf("%s: %w", referenceContext(pipelineID, laneID, slot), err)
}
if err := validateBindingLLMProfile(pipelineID, laneID, slot, binding); err != nil {
return err
}
if binding.Retries < 0 {
if laneID != "" {
return fmt.Errorf("pipeline %q lane %q %s retries must be greater than or equal to zero", pipelineID, laneID, slot)
}
return fmt.Errorf("pipeline %q %s retries must be greater than or equal to zero", pipelineID, slot)
}
if err := validateValidatorOverride(pipelineID, laneID, slot, binding.Validators); err != nil {
return err
}
@@ -197,13 +145,6 @@ func validateBinding(
return validateReferenceMapForContext(pipelineID, laneID, slot, binding.References, true)
}
func validateStructuredOutputRepairAttempts(context string, attempts *int) error {
if attempts != nil && (*attempts < 0 || *attempts > 3) {
return fmt.Errorf("%s structured_output_repair_attempts must be between zero and three", context)
}
return nil
}
func validateValidatorOverride(pipelineID string, laneID string, slot string, override pipeline.ValidatorOverride) error {
if !override.Set {
return nil
@@ -215,9 +156,6 @@ func validateValidatorOverride(pipelineID string, laneID string, slot string, ov
}
for i, validator := range override.Validators {
context := fmt.Sprintf("%s validators[%d]", referenceContext(pipelineID, laneID, slot), i)
if err := validateStructuredOutputRepairAttempts(context, validator.StructuredOutputRepairAttempts); err != nil {
return err
}
if strings.TrimSpace(validator.Module) == "" {
return fmt.Errorf("%s module must not be empty", context)
}
@@ -227,11 +165,8 @@ func validateValidatorOverride(pipelineID string, laneID string, slot string, ov
if validator.Validators.Set {
return fmt.Errorf("%s nested validators are not supported", context)
}
if validator.ValidationPolicy != nil {
return fmt.Errorf("%s validation_policy is not supported", context)
}
if validator.Retries < 0 {
return fmt.Errorf("%s retries must be greater than or equal to zero", context)
if validator.Mode != nil || validator.Cache != nil || validator.MaxAdditionalTurns != nil {
return fmt.Errorf("%s producer controls are not supported for validators", context)
}
if validator.LLMProfile != "" && strings.TrimSpace(validator.LLMProfile) == "" {
return fmt.Errorf("%s llm_profile must not be empty when set", context)

View File

@@ -8,142 +8,31 @@ import (
)
func TestValidateConcurrencyRules(t *testing.T) {
tests := []struct {
name string
setup func(*Config)
want string
}{
{
name: "non-positive total",
setup: func(cfg *Config) {
cfg.Concurrency.TotalLLM = 0
},
want: "total LLM concurrency must be greater than zero",
},
{
name: "worker below one",
setup: func(cfg *Config) {
cfg.Concurrency.TotalLLM = 3
cfg.Concurrency.StageWorkers = map[string]int{"extract": 0}
},
want: "stage_workers.extract must be between 1",
},
{
name: "worker above total",
setup: func(cfg *Config) {
cfg.Concurrency.TotalLLM = 3
cfg.Concurrency.StageWorkers = map[string]int{"extract": 4}
},
want: "stage_workers.extract must be between 1",
},
{
name: "worker lower boundary",
setup: func(cfg *Config) {
cfg.Concurrency.TotalLLM = 3
cfg.Concurrency.StageWorkers = map[string]int{"extract": 1}
},
},
{
name: "worker upper boundary",
setup: func(cfg *Config) {
cfg.Concurrency.TotalLLM = 3
cfg.Concurrency.StageWorkers = map[string]int{"extract": 3}
},
},
{
name: "unknown worker key",
setup: func(cfg *Config) {
cfg.Concurrency.StageWorkers = map[string]int{"worker": 1}
},
want: "stage_workers key \"worker\" is not supported",
},
{
name: "blank worker key",
setup: func(cfg *Config) {
cfg.Concurrency.StageWorkers = map[string]int{" ": 1}
},
want: "stage_workers key must not be empty",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg := Default()
tt.setup(&cfg)
err := cfg.Validate()
if tt.want == "" {
if err != nil {
t.Fatalf("Validate() error = %v", err)
}
return
}
if err == nil || !strings.Contains(err.Error(), tt.want) {
t.Fatalf("Validate() error = %v, want context %q", err, tt.want)
}
})
for _, workers := range []int{0, -1, 1, 64, 1000000} {
cfg := Default()
cfg.Concurrency.Workers = workers
err := cfg.Validate()
if (err != nil) != (workers <= 0) {
t.Fatalf("workers %d: %v", workers, err)
}
}
}
func TestValidateStateSurfaceRules(t *testing.T) {
tests := []struct {
for _, test := range []struct {
name string
setup func(*Config)
want string
}{
{
name: "blank output root",
setup: func(cfg *Config) {
cfg.Output.Directory = " "
},
want: "output.directory must not be empty",
},
{
name: "blank debug root",
setup: func(cfg *Config) {
cfg.Debug.Directory = " "
},
want: "debug.directory must not be empty",
},
{
name: "NUL in output root",
setup: func(cfg *Config) {
cfg.Output.Directory = "./out\x00put"
},
want: "output.directory must not contain NUL",
},
{
name: "NUL in chunk plan root",
setup: func(cfg *Config) {
cfg.Cache.ChunkPlans.Directory = "./plans\x00"
},
want: "cache.chunk_plans.directory must not contain NUL",
},
{
name: "NUL in checkpoint root",
setup: func(cfg *Config) {
cfg.Cache.Checkpoints.Directory = "./checkpoints\x00"
},
want: "cache.checkpoints.directory must not contain NUL",
},
{
name: "NUL in debug root",
setup: func(cfg *Config) {
cfg.Debug.Directory = "./debug\x00"
},
want: "debug.directory must not contain NUL",
},
{
name: "invalid chunk plan mode",
setup: func(cfg *Config) {
cfg.Cache.ChunkPlans.Mode = pipeline.ChunkCacheMode("invalid")
},
want: "cache.chunk_plans.mode:",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg := Default()
tt.setup(&cfg)
assertValidationContains(t, cfg, tt.want)
{"output", func(c *Config) { c.Output.Directory = "" }},
{"cache", func(c *Config) { c.CacheDirectory = "\\x00" }},
{"debug", func(c *Config) { c.Debug.Directory = "\\x00" }},
} {
t.Run(test.name, func(t *testing.T) {
c := Default()
test.setup(&c)
if c.Validate() == nil {
t.Fatal("accepted invalid path")
}
})
}
}
@@ -214,20 +103,21 @@ func TestValidateIdentifiersAfterTrimming(t *testing.T) {
}
}
func TestValidateBindingRetriesAndProfiles(t *testing.T) {
func TestValidateBindingControlsAndProfiles(t *testing.T) {
tests := []struct {
name string
setup func(*Config)
want string
}{
{
name: "negative retries",
name: "negative turns",
setup: func(cfg *Config) {
profile := validationProfile()
profile.Input.Retries = -1
value := -1
profile.Input.MaxAdditionalTurns = &value
cfg.Pipelines = map[string]pipeline.PipelineProfile{"main": profile}
},
want: "input retries must be greater than or equal to zero",
want: "input: max_additional_turns must not be negative",
},
{
name: "whitespace-only input profile",
@@ -302,12 +192,12 @@ func TestValidateValidatorBindingRules(t *testing.T) {
profile.Chunk.Validators = pipeline.ValidatorOverride{
Set: true,
Validators: []pipeline.ModuleBinding{{
Module: "validator",
Retries: -1,
Module: "validator",
MaxAdditionalTurns: documentValue(-1),
}},
}
},
want: "chunk validators[0] retries must be greater than or equal to zero",
want: "chunk validators[0] producer controls are not supported for validators",
},
{
name: "validator references",

View File

@@ -1,103 +1,67 @@
package config
import (
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
configkit "go.maximumdirect.net/configkit"
)
func TestValidationPolicyFileConfigurationIsStrictAndPresenceAware(t *testing.T) {
const valid = `version: 6
func TestLoadedProducerControlsPreserveOmissionAndZero(t *testing.T) {
t.Parallel()
loaded, err := loadConfigSource(configkit.Bytes("controls.yml", []byte(`version: 6
pipelines:
main:
validation_policy:
producer_structural_failure: reject_output
semantic_rejection: fail_run
input:
module: seriatim
mode: " permissive "
max_additional_turns: 5
cache: refresh
input: {module: input}
chunk:
module: generic
validation_policy:
validator_failure: fail_run
module: chunk
mode: strict
max_additional_turns: 0
cache: bypass
artifacts:
lane:
extract:
module: extract
validation_policy:
semantic_rejection: reject_output
`
loaded, err := loadDocumentSource(configkit.Bytes("valid.yml", []byte(valid)), LoadOptions{})
extract: {module: extract}
`)), LoadOptions{})
if err != nil {
t.Fatal(err)
}
cfg := loaded.Config
profile := cfg.Pipelines["main"]
if profile.ValidationPolicy == nil || profile.ValidationPolicy.ProducerStructuralFailure == nil || *profile.ValidationPolicy.ProducerStructuralFailure != pipeline.ProducerStructuralFailureRejectOutput || profile.ValidationPolicy.SemanticRejection == nil || *profile.ValidationPolicy.SemanticRejection != pipeline.SemanticRejectionFailRun || profile.ValidationPolicy.ValidatorFailure != nil {
t.Fatalf("pipeline validation policy = %#v", profile.ValidationPolicy)
profile := loaded.Config.Pipelines["main"]
if profile.Mode == nil || *profile.Mode != pipeline.ModePermissive {
t.Fatalf("mode = %v", profile.Mode)
}
if profile.Chunk.ValidationPolicy == nil || profile.Chunk.ValidationPolicy.ValidatorFailure == nil || *profile.Chunk.ValidationPolicy.ValidatorFailure != pipeline.ValidatorFailureFailRun {
t.Fatalf("chunk validation policy = %#v", profile.Chunk.ValidationPolicy)
if profile.Artifacts["lane"].Extract.Mode != nil || profile.Artifacts["lane"].Extract.MaxAdditionalTurns != nil {
t.Fatal("omitted binding controls became explicit")
}
lane := profile.Artifacts["lane"]
if lane.Extract.ValidationPolicy == nil || lane.Extract.ValidationPolicy.SemanticRejection == nil || *lane.Extract.ValidationPolicy.SemanticRejection != pipeline.SemanticRejectionRejectOutput {
t.Fatalf("extract validation policy = %#v", lane.Extract.ValidationPolicy)
got, err := pipeline.ResolveProducerControls(profile.Chunk, profile, nil)
if err != nil {
t.Fatal(err)
}
for _, test := range []struct {
name string
yaml string
}{
{"null object", strings.Replace(valid, "validation_policy:\n producer_structural_failure: reject_output\n semantic_rejection: fail_run", "validation_policy: null", 1)},
{"null field", strings.Replace(valid, "semantic_rejection: fail_run", "semantic_rejection: null", 1)},
{"unknown field", strings.Replace(valid, "semantic_rejection: fail_run", "unknown: fail_run", 1)},
{"duplicate field", strings.Replace(valid, "semantic_rejection: fail_run", "semantic_rejection: fail_run\n semantic_rejection: reject_output", 1)},
{"invalid enum", strings.Replace(valid, "semantic_rejection: fail_run", "semantic_rejection: continue", 1)},
} {
t.Run(test.name, func(t *testing.T) {
if _, err := loadDocumentSource(configkit.Bytes("invalid.yml", []byte(test.yaml)), LoadOptions{}); err == nil {
t.Fatal("loadDocumentSource() error = nil, want strict validation-policy rejection")
}
})
if got != (pipeline.ProducerControls{Mode: pipeline.ModeStrict, MaxAdditionalTurns: 0, Cache: pipeline.CacheBypass}) {
t.Fatalf("chunk controls = %#v", got)
}
got, err = pipeline.ResolveProducerControls(profile.Artifacts["lane"].Extract, profile, nil)
if err != nil {
t.Fatal(err)
}
if got != (pipeline.ProducerControls{Mode: pipeline.ModePermissive, MaxAdditionalTurns: 5, Cache: pipeline.CacheRefresh}) {
t.Fatalf("inherited controls = %#v", got)
}
invocation := pipeline.CacheAuto
got, err = pipeline.ResolveProducerControls(profile.Chunk, profile, &invocation)
if err != nil || got.Cache != pipeline.CacheAuto {
t.Fatalf("invocation cache = %#v, %v", got, err)
}
}
func TestValidationPolicyPlacementRules(t *testing.T) {
policy := &pipeline.ValidationPolicyOverride{}
semantic := pipeline.SemanticRejectionRejectOutput
policy.SemanticRejection = &semantic
base := pipeline.PipelineProfile{
ID: "main",
Input: pipeline.Binding("input"),
Artifacts: map[string]pipeline.ArtifactLaneProfile{
"lane": {Extract: pipeline.Binding("extract")},
},
}
for _, test := range []struct {
name string
mutate func(*pipeline.PipelineProfile)
}{
{"input", func(profile *pipeline.PipelineProfile) { profile.Input.ValidationPolicy = policy }},
{"output", func(profile *pipeline.PipelineProfile) {
profile.Output = pipeline.Binding("output")
profile.Output.ValidationPolicy = policy
}},
{"validator", func(profile *pipeline.PipelineProfile) {
lane := profile.Artifacts["lane"]
lane.Extract.Validators = pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{{Module: "validator", ValidationPolicy: policy}}}
profile.Artifacts["lane"] = lane
}},
func TestLoadedProducerControlsValidateAtTheirPaths(t *testing.T) {
t.Parallel()
for _, test := range []struct{ field, value string }{
{"mode", "unknown"}, {"cache", "unknown"}, {"max_additional_turns", "-1"},
} {
t.Run(test.name, func(t *testing.T) {
profile := base
profile.Artifacts = map[string]pipeline.ArtifactLaneProfile{"lane": base.Artifacts["lane"]}
test.mutate(&profile)
cfg := Default()
cfg.Pipelines = map[string]pipeline.PipelineProfile{"main": profile}
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "validation_policy") {
t.Fatalf("Config.Validate() error = %v, want placement rejection", err)
}
})
_, err := loadConfigSource(configkit.Bytes("invalid.yml", []byte("version: 6\npipelines:\n main:\n input: {module: input}\n "+test.field+": "+test.value+"\n")), LoadOptions{})
assertDocumentDiagnosticAt(t, err, "/pipelines/main/"+test.field)
}
}