Define typed producer assessment and native execution dependencies
This commit is contained in:
@@ -367,6 +367,19 @@ one explicit smoke outcome. Full pipeline binding is completed in Stage 33.
|
|||||||
|
|
||||||
## Stage 8: Define Typed Producer Collaboration And Resource Dependencies
|
## Stage 8: Define Typed Producer Collaboration And Resource Dependencies
|
||||||
|
|
||||||
|
**Status:** Complete. Construction now passes the shared native engine. Typed
|
||||||
|
producer requests carry explicit assessment callbacks, resolved controls, native
|
||||||
|
execution options, and incoming reuse eligibility; results carry accepted/fallback
|
||||||
|
disposition, reason, and eligibility without a completion facade. Validators receive
|
||||||
|
native execution options only. Static prompt-ID dependencies replace correction
|
||||||
|
transport capabilities and are copied through registration, resolution, and
|
||||||
|
preparation. Exact artifact type, codec/evidence, option, and reference checks remain.
|
||||||
|
Added registry callback/native-option, incompatible-type, dependency-isolation, and
|
||||||
|
preparation coverage. Contracts tests/vet and `git diff --check` pass. Pipeline
|
||||||
|
tests remain blocked by old ConfigKit's removed YAML `plugin/limit` import; its
|
||||||
|
production build also encounters unmigrated policy/normalize-retry consumers.
|
||||||
|
Their scheduled migration remains required; blocked tests are not claimed passing.
|
||||||
|
|
||||||
**Goal:** expose only the application contracts required by native execution.
|
**Goal:** expose only the application contracts required by native execution.
|
||||||
|
|
||||||
**Inspect/change:** `internal/framework/contracts/{contracts,typed_pipeline}.go`,
|
**Inspect/change:** `internal/framework/contracts/{contracts,typed_pipeline}.go`,
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
package config
|
package config
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
"go.maximumdirect.net/promptkit"
|
"go.maximumdirect.net/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -19,9 +19,9 @@ type Document struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type PipelineDocument struct {
|
type PipelineDocument struct {
|
||||||
Mode *pipeline.ExecutionMode `yaml:"mode,omitempty"`
|
Mode *contracts.ExecutionMode `yaml:"mode,omitempty"`
|
||||||
MaxAdditionalTurns *int `yaml:"max_additional_turns,omitempty"`
|
MaxAdditionalTurns *int `yaml:"max_additional_turns,omitempty"`
|
||||||
Cache *pipeline.CacheMode `yaml:"cache,omitempty"`
|
Cache *contracts.CacheMode `yaml:"cache,omitempty"`
|
||||||
LLMProfile *string `yaml:"llm_profile,omitempty"`
|
LLMProfile *string `yaml:"llm_profile,omitempty"`
|
||||||
Input BindingDocument `yaml:"input"`
|
Input BindingDocument `yaml:"input"`
|
||||||
Chunk *BindingDocument `yaml:"chunk,omitempty"`
|
Chunk *BindingDocument `yaml:"chunk,omitempty"`
|
||||||
@@ -49,9 +49,9 @@ type ArtifactLaneDocument struct {
|
|||||||
// deliberately finite subset so nested validator-only fields are rejected by
|
// deliberately finite subset so nested validator-only fields are rejected by
|
||||||
// ConfigKit's strict decoder.
|
// ConfigKit's strict decoder.
|
||||||
type BindingDocument struct {
|
type BindingDocument struct {
|
||||||
Mode *pipeline.ExecutionMode `yaml:"mode,omitempty"`
|
Mode *contracts.ExecutionMode `yaml:"mode,omitempty"`
|
||||||
MaxAdditionalTurns *int `yaml:"max_additional_turns,omitempty"`
|
MaxAdditionalTurns *int `yaml:"max_additional_turns,omitempty"`
|
||||||
Cache *pipeline.CacheMode `yaml:"cache,omitempty"`
|
Cache *contracts.CacheMode `yaml:"cache,omitempty"`
|
||||||
Module string `yaml:"module"`
|
Module string `yaml:"module"`
|
||||||
LLMProfile *string `yaml:"llm_profile,omitempty"`
|
LLMProfile *string `yaml:"llm_profile,omitempty"`
|
||||||
Options map[string]any `yaml:"options,omitempty"`
|
Options map[string]any `yaml:"options,omitempty"`
|
||||||
|
|||||||
@@ -5,11 +5,12 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
)
|
)
|
||||||
|
|
||||||
type ResolveInput struct {
|
type ResolveInput struct {
|
||||||
CacheOverride *pipeline.CacheMode
|
CacheOverride *contracts.CacheMode
|
||||||
PipelineID string
|
PipelineID string
|
||||||
Only []string
|
Only []string
|
||||||
Catalog pipeline.ModuleCatalog
|
Catalog pipeline.ModuleCatalog
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
configkit "go.maximumdirect.net/configkit"
|
configkit "go.maximumdirect.net/configkit"
|
||||||
)
|
)
|
||||||
@@ -449,21 +450,21 @@ func sortedDocumentKeys[T any](values map[string]T) []string {
|
|||||||
sort.Strings(keys)
|
sort.Strings(keys)
|
||||||
return keys
|
return keys
|
||||||
}
|
}
|
||||||
func normalizeDocumentControls(mode **pipeline.ExecutionMode, cache **pipeline.CacheMode, path configkit.Path, resolution documentNormalizer) {
|
func normalizeDocumentControls(mode **contracts.ExecutionMode, cache **contracts.CacheMode, path configkit.Path, resolution documentNormalizer) {
|
||||||
if *mode != nil {
|
if *mode != nil {
|
||||||
value := pipeline.ExecutionMode(strings.TrimSpace(string(**mode)))
|
value := contracts.ExecutionMode(strings.TrimSpace(string(**mode)))
|
||||||
if **mode != value {
|
if **mode != value {
|
||||||
normalizeDocumentValue(resolution, path.Key("mode"), mode, &value, "trimmed execution mode")
|
normalizeDocumentValue(resolution, path.Key("mode"), mode, &value, "trimmed execution mode")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if *cache != nil {
|
if *cache != nil {
|
||||||
value := pipeline.CacheMode(strings.TrimSpace(string(**cache)))
|
value := contracts.CacheMode(strings.TrimSpace(string(**cache)))
|
||||||
if **cache != value {
|
if **cache != value {
|
||||||
normalizeDocumentValue(resolution, path.Key("cache"), cache, &value, "trimmed cache mode")
|
normalizeDocumentValue(resolution, path.Key("cache"), cache, &value, "trimmed cache mode")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
func validateDocumentControls(mode *pipeline.ExecutionMode, turns *int, cache *pipeline.CacheMode, path configkit.Path, validation *configkit.Validation) {
|
func validateDocumentControls(mode *contracts.ExecutionMode, turns *int, cache *contracts.CacheMode, path configkit.Path, validation *configkit.Validation) {
|
||||||
if mode != nil {
|
if mode != nil {
|
||||||
if _, err := pipeline.ParseExecutionMode(string(*mode)); err != nil {
|
if _, err := pipeline.ParseExecutionMode(string(*mode)); err != nil {
|
||||||
validation.Error(path.Key("mode"), err.Error())
|
validation.Error(path.Key("mode"), err.Error())
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package config
|
|||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
configkit "go.maximumdirect.net/configkit"
|
configkit "go.maximumdirect.net/configkit"
|
||||||
)
|
)
|
||||||
@@ -29,7 +30,7 @@ pipelines:
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
profile := loaded.Config.Pipelines["main"]
|
profile := loaded.Config.Pipelines["main"]
|
||||||
if profile.Mode == nil || *profile.Mode != pipeline.ModePermissive {
|
if profile.Mode == nil || *profile.Mode != contracts.ModePermissive {
|
||||||
t.Fatalf("mode = %v", profile.Mode)
|
t.Fatalf("mode = %v", profile.Mode)
|
||||||
}
|
}
|
||||||
if profile.Artifacts["lane"].Extract.Mode != nil || profile.Artifacts["lane"].Extract.MaxAdditionalTurns != nil {
|
if profile.Artifacts["lane"].Extract.Mode != nil || profile.Artifacts["lane"].Extract.MaxAdditionalTurns != nil {
|
||||||
@@ -39,19 +40,19 @@ pipelines:
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if got != (pipeline.ProducerControls{Mode: pipeline.ModeStrict, MaxAdditionalTurns: 0, Cache: pipeline.CacheBypass}) {
|
if got != (contracts.ProducerControls{Mode: contracts.ModeStrict, MaxAdditionalTurns: 0, Cache: contracts.CacheBypass}) {
|
||||||
t.Fatalf("chunk controls = %#v", got)
|
t.Fatalf("chunk controls = %#v", got)
|
||||||
}
|
}
|
||||||
got, err = pipeline.ResolveProducerControls(profile.Artifacts["lane"].Extract, profile, nil)
|
got, err = pipeline.ResolveProducerControls(profile.Artifacts["lane"].Extract, profile, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if got != (pipeline.ProducerControls{Mode: pipeline.ModePermissive, MaxAdditionalTurns: 5, Cache: pipeline.CacheRefresh}) {
|
if got != (contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 5, Cache: contracts.CacheRefresh}) {
|
||||||
t.Fatalf("inherited controls = %#v", got)
|
t.Fatalf("inherited controls = %#v", got)
|
||||||
}
|
}
|
||||||
invocation := pipeline.CacheAuto
|
invocation := contracts.CacheAuto
|
||||||
got, err = pipeline.ResolveProducerControls(profile.Chunk, profile, &invocation)
|
got, err = pipeline.ResolveProducerControls(profile.Chunk, profile, &invocation)
|
||||||
if err != nil || got.Cache != pipeline.CacheAuto {
|
if err != nil || got.Cache != contracts.CacheAuto {
|
||||||
t.Fatalf("invocation cache = %#v, %v", got, err)
|
t.Fatalf("invocation cache = %#v, %v", got, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -181,20 +181,19 @@ type InputAdapter interface {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type ChunkRequest struct {
|
type ChunkRequest struct {
|
||||||
|
Operation ProducerOperation[source.ChunkPlan] `json:"-"`
|
||||||
Source *source.SourceDocument `json:"-"`
|
Source *source.SourceDocument `json:"-"`
|
||||||
SourceInput LLMInputMaterial `json:"source_input,omitempty"`
|
SourceInput LLMInputMaterial `json:"source_input,omitempty"`
|
||||||
SessionID string `json:"session_id,omitempty"`
|
SessionID string `json:"session_id,omitempty"`
|
||||||
References ReferenceSet `json:"references,omitempty"`
|
References ReferenceSet `json:"references,omitempty"`
|
||||||
LLMProfile string `json:"llm_profile,omitempty"`
|
LLMProfile string `json:"llm_profile,omitempty"`
|
||||||
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
|
|
||||||
Correction *SemanticCorrection `json:"-"`
|
|
||||||
Metadata map[string]any `json:"metadata,omitempty"`
|
Metadata map[string]any `json:"metadata,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type ChunkPlanResult struct {
|
type ChunkPlanResult struct {
|
||||||
|
Outcome ProducerOutcome `json:"outcome"`
|
||||||
Plan source.ChunkPlan `json:"plan"`
|
Plan source.ChunkPlan `json:"plan"`
|
||||||
Diagnostics []ProducerDiagnostic `json:"diagnostics,omitempty"`
|
Diagnostics []ProducerDiagnostic `json:"diagnostics,omitempty"`
|
||||||
ModelCandidate *ModelCandidate `json:"-"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type Chunker interface {
|
type Chunker interface {
|
||||||
|
|||||||
@@ -278,7 +278,7 @@ func (extractor fakeExtractor) ReferenceSlots() []ReferenceSlot {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (extractor fakeExtractor) Extract(ctx context.Context, req TypedExtractionRequest) (TypedExtractionResult[fakeArtifact], error) {
|
func (extractor fakeExtractor) Extract(ctx context.Context, req TypedExtractionRequest[fakeArtifact]) (TypedExtractionResult[fakeArtifact], error) {
|
||||||
value := "example"
|
value := "example"
|
||||||
if req.AmbientContext["mode"] == "chunked" {
|
if req.AmbientContext["mode"] == "chunked" {
|
||||||
value = "chunked"
|
value = "chunked"
|
||||||
|
|||||||
87
internal/framework/contracts/producer.go
Normal file
87
internal/framework/contracts/producer.go
Normal file
@@ -0,0 +1,87 @@
|
|||||||
|
package contracts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
type ExecutionMode string
|
||||||
|
|
||||||
|
const (
|
||||||
|
ModeStrict ExecutionMode = "strict"
|
||||||
|
ModePermissive ExecutionMode = "permissive"
|
||||||
|
)
|
||||||
|
|
||||||
|
type CacheMode string
|
||||||
|
|
||||||
|
const (
|
||||||
|
CacheAuto CacheMode = "auto"
|
||||||
|
CacheBypass CacheMode = "bypass"
|
||||||
|
CacheRefresh CacheMode = "refresh"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ProducerControls is resolved before execution; modules do not inherit settings.
|
||||||
|
type ProducerControls struct {
|
||||||
|
Mode ExecutionMode `json:"mode"`
|
||||||
|
MaxAdditionalTurns int `json:"max_additional_turns"`
|
||||||
|
Cache CacheMode `json:"cache"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExecutionOptions carries native request options, not a completion protocol.
|
||||||
|
// Observer closures shared by concurrent operations must synchronize their state.
|
||||||
|
type ExecutionOptions struct {
|
||||||
|
Execution *promptkit.ExecutionTargetOverride
|
||||||
|
TransientRetries *int
|
||||||
|
Observer promptkit.ExecutionObserver `json:"-"`
|
||||||
|
OnPrepared func(context.Context, promptkit.PreparedRun) `json:"-"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type AssessmentDecision string
|
||||||
|
|
||||||
|
const (
|
||||||
|
AssessmentApproved AssessmentDecision = "approved"
|
||||||
|
AssessmentRejected AssessmentDecision = "rejected"
|
||||||
|
AssessmentFailed AssessmentDecision = "failed"
|
||||||
|
AssessmentUnavailable AssessmentDecision = "unavailable"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ValidatorAssessment keeps model-facing guidance in Result separate from
|
||||||
|
// diagnostic reason codes, operator prose, and operational failures.
|
||||||
|
type ValidatorAssessment struct {
|
||||||
|
ValidatorName string
|
||||||
|
Decision AssessmentDecision
|
||||||
|
Result ValidationResult
|
||||||
|
Failure error
|
||||||
|
}
|
||||||
|
|
||||||
|
// CandidateAssessment contains the complete ordered chain for one candidate.
|
||||||
|
type CandidateAssessment struct {
|
||||||
|
Validators []ValidatorAssessment
|
||||||
|
}
|
||||||
|
|
||||||
|
type CandidateAssessor[T any] func(context.Context, T) (CandidateAssessment, error)
|
||||||
|
|
||||||
|
// ProducerOperation is supplied explicitly by the runner for one operation.
|
||||||
|
// Validators receive ExecutionOptions, never a producer assessment callback.
|
||||||
|
type ProducerOperation[T any] struct {
|
||||||
|
Controls ProducerControls
|
||||||
|
Assess CandidateAssessor[T] `json:"-"`
|
||||||
|
Execution ExecutionOptions
|
||||||
|
ReuseEligible bool
|
||||||
|
}
|
||||||
|
|
||||||
|
type ProducerDisposition string
|
||||||
|
|
||||||
|
const (
|
||||||
|
ProducerAccepted ProducerDisposition = "accepted"
|
||||||
|
ProducerFallback ProducerDisposition = "fallback"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ProducerOutcome qualifies a returned domain value without wrapping a native
|
||||||
|
// execution result. A zero outcome is not accepted or eligible for reuse.
|
||||||
|
type ProducerOutcome struct {
|
||||||
|
Disposition ProducerDisposition `json:"disposition"`
|
||||||
|
FallbackReason string `json:"fallback_reason,omitempty"`
|
||||||
|
ReuseEligible bool `json:"reuse_eligible"`
|
||||||
|
}
|
||||||
@@ -33,7 +33,8 @@ type NormalizeArtifact[T any] struct {
|
|||||||
Value T
|
Value T
|
||||||
}
|
}
|
||||||
|
|
||||||
type TypedExtractionRequest struct {
|
type TypedExtractionRequest[T any] struct {
|
||||||
|
Operation ProducerOperation[T]
|
||||||
Source *source.SourceDocument
|
Source *source.SourceDocument
|
||||||
Chunk *source.Chunk
|
Chunk *source.Chunk
|
||||||
AmbientContext map[string]any
|
AmbientContext map[string]any
|
||||||
@@ -41,24 +42,23 @@ type TypedExtractionRequest struct {
|
|||||||
SessionID string
|
SessionID string
|
||||||
References ReferenceSet
|
References ReferenceSet
|
||||||
LLMProfile string
|
LLMProfile string
|
||||||
StructuredOutputRepairAttempts *int
|
|
||||||
Correction *SemanticCorrection
|
|
||||||
Metadata map[string]any
|
Metadata map[string]any
|
||||||
}
|
}
|
||||||
|
|
||||||
type TypedExtractionResult[T any] struct {
|
type TypedExtractionResult[T any] struct {
|
||||||
|
Outcome ProducerOutcome
|
||||||
Value T
|
Value T
|
||||||
Diagnostics []ProducerDiagnostic
|
Diagnostics []ProducerDiagnostic
|
||||||
ModelCandidate *ModelCandidate
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type Extractor[T any] interface {
|
type Extractor[T any] interface {
|
||||||
Key() string
|
Key() string
|
||||||
ReferenceSlots() []ReferenceSlot
|
ReferenceSlots() []ReferenceSlot
|
||||||
Extract(context.Context, TypedExtractionRequest) (TypedExtractionResult[T], error)
|
Extract(context.Context, TypedExtractionRequest[T]) (TypedExtractionResult[T], error)
|
||||||
}
|
}
|
||||||
|
|
||||||
type TypedMergeRequest[T any] struct {
|
type TypedMergeRequest[T any] struct {
|
||||||
|
Operation ProducerOperation[T]
|
||||||
Source *source.SourceDocument
|
Source *source.SourceDocument
|
||||||
LaneID string
|
LaneID string
|
||||||
ExtractOutputs []ExtractArtifact[T]
|
ExtractOutputs []ExtractArtifact[T]
|
||||||
@@ -66,15 +66,13 @@ type TypedMergeRequest[T any] struct {
|
|||||||
SessionID string
|
SessionID string
|
||||||
References ReferenceSet
|
References ReferenceSet
|
||||||
LLMProfile string
|
LLMProfile string
|
||||||
StructuredOutputRepairAttempts *int
|
|
||||||
Correction *SemanticCorrection
|
|
||||||
Metadata map[string]any
|
Metadata map[string]any
|
||||||
}
|
}
|
||||||
|
|
||||||
type TypedMergeResult[T any] struct {
|
type TypedMergeResult[T any] struct {
|
||||||
|
Outcome ProducerOutcome
|
||||||
Value T
|
Value T
|
||||||
Diagnostics []ProducerDiagnostic
|
Diagnostics []ProducerDiagnostic
|
||||||
ModelCandidate *ModelCandidate
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type Merger[T any] interface {
|
type Merger[T any] interface {
|
||||||
@@ -83,6 +81,7 @@ type Merger[T any] interface {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type TypedNormalizeRequest[T any] struct {
|
type TypedNormalizeRequest[T any] struct {
|
||||||
|
Operation ProducerOperation[T]
|
||||||
Source *source.SourceDocument
|
Source *source.SourceDocument
|
||||||
LaneID string
|
LaneID string
|
||||||
MergeOutput MergeArtifact[T]
|
MergeOutput MergeArtifact[T]
|
||||||
@@ -90,33 +89,13 @@ type TypedNormalizeRequest[T any] struct {
|
|||||||
SessionID string
|
SessionID string
|
||||||
References ReferenceSet
|
References ReferenceSet
|
||||||
LLMProfile string
|
LLMProfile string
|
||||||
StructuredOutputRepairAttempts *int
|
|
||||||
Correction *SemanticCorrection
|
|
||||||
Metadata map[string]any
|
Metadata map[string]any
|
||||||
}
|
}
|
||||||
|
|
||||||
type TypedNormalizeResult[T any] struct {
|
type TypedNormalizeResult[T any] struct {
|
||||||
|
Outcome ProducerOutcome
|
||||||
Value T
|
Value T
|
||||||
Diagnostics []ProducerDiagnostic
|
Diagnostics []ProducerDiagnostic
|
||||||
Retry *NormalizeRetry
|
|
||||||
ModelCandidate *ModelCandidate
|
|
||||||
}
|
|
||||||
|
|
||||||
// Normalize retry limits bound module-provided control and diagnostic text
|
|
||||||
// before the framework consumes or records it.
|
|
||||||
const (
|
|
||||||
MaxNormalizeRetryReasonCodeBytes = 128
|
|
||||||
MaxNormalizeRetryMessageBytes = 4096
|
|
||||||
MaxNormalizeRetryCorrectionGuidanceBytes = 4096
|
|
||||||
)
|
|
||||||
|
|
||||||
// NormalizeRetry asks the framework to retry normalization while retaining a
|
|
||||||
// safe candidate for acceptance if the retry budget is exhausted.
|
|
||||||
type NormalizeRetry struct {
|
|
||||||
ReasonCode string
|
|
||||||
Message string
|
|
||||||
CorrectionGuidance string
|
|
||||||
FallbackDiagnostics []ProducerDiagnostic
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type Normalizer[T any] interface {
|
type Normalizer[T any] interface {
|
||||||
@@ -126,6 +105,7 @@ type Normalizer[T any] interface {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type TypedValidationRequest[T any] struct {
|
type TypedValidationRequest[T any] struct {
|
||||||
|
Execution ExecutionOptions
|
||||||
Stage string
|
Stage string
|
||||||
LaneID string
|
LaneID string
|
||||||
ModuleKey string
|
ModuleKey string
|
||||||
@@ -135,7 +115,6 @@ type TypedValidationRequest[T any] struct {
|
|||||||
SessionID string
|
SessionID string
|
||||||
References ReferenceSet
|
References ReferenceSet
|
||||||
LLMProfile string
|
LLMProfile string
|
||||||
StructuredOutputRepairAttempts *int
|
|
||||||
Metadata map[string]any
|
Metadata map[string]any
|
||||||
Chunk *source.Chunk
|
Chunk *source.Chunk
|
||||||
Chunks []source.Chunk
|
Chunks []source.Chunk
|
||||||
@@ -150,6 +129,7 @@ type TypedValidator[T any] interface {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type ChunkValidationRequest struct {
|
type ChunkValidationRequest struct {
|
||||||
|
Execution ExecutionOptions
|
||||||
ModuleKey string
|
ModuleKey string
|
||||||
Source *source.SourceDocument
|
Source *source.SourceDocument
|
||||||
SourceID string
|
SourceID string
|
||||||
@@ -157,7 +137,6 @@ type ChunkValidationRequest struct {
|
|||||||
SessionID string
|
SessionID string
|
||||||
References ReferenceSet
|
References ReferenceSet
|
||||||
LLMProfile string
|
LLMProfile string
|
||||||
StructuredOutputRepairAttempts *int
|
|
||||||
Metadata map[string]any
|
Metadata map[string]any
|
||||||
Chunks []source.Chunk
|
Chunks []source.Chunk
|
||||||
}
|
}
|
||||||
@@ -169,6 +148,7 @@ type ChunkValidator interface {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type SerializedValidationRequest struct {
|
type SerializedValidationRequest struct {
|
||||||
|
Execution ExecutionOptions
|
||||||
Stage string
|
Stage string
|
||||||
LaneID string
|
LaneID string
|
||||||
ModuleKey string
|
ModuleKey string
|
||||||
@@ -178,7 +158,6 @@ type SerializedValidationRequest struct {
|
|||||||
SessionID string
|
SessionID string
|
||||||
References ReferenceSet
|
References ReferenceSet
|
||||||
LLMProfile string
|
LLMProfile string
|
||||||
StructuredOutputRepairAttempts *int
|
|
||||||
Metadata map[string]any
|
Metadata map[string]any
|
||||||
Chunk *source.Chunk
|
Chunk *source.Chunk
|
||||||
Chunks []source.Chunk
|
Chunks []source.Chunk
|
||||||
|
|||||||
@@ -6,12 +6,13 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ModuleDependencies contains run-scoped collaborators shared by constructed
|
// ModuleDependencies contains run-scoped collaborators shared by constructed
|
||||||
// modules. Implementations retain only the dependencies they use.
|
// modules. Implementations retain only the dependencies they use.
|
||||||
type ModuleDependencies struct {
|
type ModuleDependencies struct {
|
||||||
LLM contracts.StructuredLLMClient
|
Engine *promptkit.Engine
|
||||||
}
|
}
|
||||||
|
|
||||||
// BuildRequest contains the stable dependencies and configured options used to
|
// BuildRequest contains the stable dependencies and configured options used to
|
||||||
|
|||||||
@@ -53,25 +53,20 @@ func RegisterExtractorBuilder[T any](registry *ExtractorRegistry, spec ModuleSpe
|
|||||||
if _, ok := registry.specs[normalized.Key]; ok {
|
if _, ok := registry.specs[normalized.Key]; ok {
|
||||||
return fmt.Errorf("extractor %q is already registered", normalized.Key)
|
return fmt.Errorf("extractor %q is already registered", normalized.Key)
|
||||||
}
|
}
|
||||||
entry := typedExtractorEntry{spec: cloneModuleSpec(normalized), valueType: reflect.TypeFor[T](), validateOptions: validateOptions, builder: func(request BuildRequest) (any, error) { return builder(cloneBuildRequest(request)) }, extract: func(ctx context.Context, implementation any, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
entry := typedExtractorEntry{spec: cloneModuleSpec(normalized), valueType: reflect.TypeFor[T](), validateOptions: validateOptions, builder: func(request BuildRequest) (any, error) { return builder(cloneBuildRequest(request)) }, extract: func(ctx context.Context, implementation any, request contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
|
||||||
extractor, ok := implementation.(contracts.Extractor[T])
|
extractor, ok := implementation.(contracts.Extractor[T])
|
||||||
if !ok {
|
if !ok {
|
||||||
return erasedTypedResult{}, fmt.Errorf("extractor %q has incompatible implementation %T", normalized.Key, implementation)
|
return erasedTypedResult{}, fmt.Errorf("extractor %q has incompatible implementation %T", normalized.Key, implementation)
|
||||||
}
|
}
|
||||||
correction, err := contracts.CloneSemanticCorrection(request.Correction)
|
result, err := extractor.Extract(ctx, contracts.TypedExtractionRequest[T]{
|
||||||
if err != nil {
|
Source: request.Source, Chunk: request.Chunk, AmbientContext: request.AmbientContext,
|
||||||
return erasedTypedResult{}, fmt.Errorf("clone extraction correction: %w", err)
|
SourceInput: request.SourceInput, SessionID: request.SessionID, References: request.References,
|
||||||
}
|
LLMProfile: request.LLMProfile, Metadata: request.Metadata, Operation: specializeProducerOperation[T](request.Operation),
|
||||||
request.Correction = correction
|
})
|
||||||
result, err := extractor.Extract(ctx, request)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return erasedTypedResult{}, err
|
return erasedTypedResult{}, err
|
||||||
}
|
}
|
||||||
candidate, err := contracts.CloneModelCandidate(result.ModelCandidate)
|
return erasedTypedResult{Value: result.Value, Diagnostics: contracts.CloneProducerDiagnostics(result.Diagnostics), Outcome: result.Outcome}, nil
|
||||||
if err != nil {
|
|
||||||
return erasedTypedResult{}, fmt.Errorf("clone extraction model candidate: %w", err)
|
|
||||||
}
|
|
||||||
return erasedTypedResult{Value: result.Value, Diagnostics: contracts.CloneProducerDiagnostics(result.Diagnostics), ModelCandidate: candidate}, nil
|
|
||||||
}}
|
}}
|
||||||
if registry.typedEntries == nil {
|
if registry.typedEntries == nil {
|
||||||
registry.typedEntries = map[string]typedExtractorEntry{}
|
registry.typedEntries = map[string]typedExtractorEntry{}
|
||||||
|
|||||||
@@ -85,19 +85,14 @@ func RegisterMergerBuilder[T any](registry *MergerRegistry, spec ModuleSpec, val
|
|||||||
}
|
}
|
||||||
outputs[i] = contracts.ExtractArtifact[T]{LaneID: output.LaneID, ExtractorKey: output.ExtractorKey, SourceID: output.SourceID, ChunkID: output.ChunkID, ChunkIndex: output.ChunkIndex, ChunkRef: output.ChunkRef, Value: value}
|
outputs[i] = contracts.ExtractArtifact[T]{LaneID: output.LaneID, ExtractorKey: output.ExtractorKey, SourceID: output.SourceID, ChunkID: output.ChunkID, ChunkIndex: output.ChunkIndex, ChunkRef: output.ChunkRef, Value: value}
|
||||||
}
|
}
|
||||||
correction, err := contracts.CloneSemanticCorrection(request.Correction)
|
result, err := merger.Merge(ctx, contracts.TypedMergeRequest[T]{
|
||||||
if err != nil {
|
Source: request.Source, LaneID: request.LaneID, ExtractOutputs: outputs,
|
||||||
return erasedTypedResult{}, fmt.Errorf("clone merge correction: %w", err)
|
SourceInput: request.SourceInput, SessionID: request.SessionID, References: request.References, LLMProfile: request.LLMProfile, Metadata: request.Metadata, Operation: specializeProducerOperation[T](request.Operation),
|
||||||
}
|
})
|
||||||
result, err := merger.Merge(ctx, contracts.TypedMergeRequest[T]{Source: request.Source, LaneID: request.LaneID, ExtractOutputs: outputs, SourceInput: request.SourceInput, SessionID: request.SessionID, References: request.References, LLMProfile: request.LLMProfile, StructuredOutputRepairAttempts: request.StructuredOutputRepairAttempts, Correction: correction, Metadata: request.Metadata})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return erasedTypedResult{}, err
|
return erasedTypedResult{}, err
|
||||||
}
|
}
|
||||||
candidate, err := contracts.CloneModelCandidate(result.ModelCandidate)
|
return erasedTypedResult{Value: result.Value, Diagnostics: contracts.CloneProducerDiagnostics(result.Diagnostics), Outcome: result.Outcome}, nil
|
||||||
if err != nil {
|
|
||||||
return erasedTypedResult{}, fmt.Errorf("clone merge model candidate: %w", err)
|
|
||||||
}
|
|
||||||
return erasedTypedResult{Value: result.Value, Diagnostics: contracts.CloneProducerDiagnostics(result.Diagnostics), ModelCandidate: candidate}, nil
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -24,7 +24,9 @@ type ModuleSpec struct {
|
|||||||
Key string
|
Key string
|
||||||
Stage ModuleStage
|
Stage ModuleStage
|
||||||
ExecutionClass contracts.ExecutionClass
|
ExecutionClass contracts.ExecutionClass
|
||||||
CorrectionProtocol contracts.CorrectionProtocol
|
// Prompts lists statically selected initial/correction IDs shared with the
|
||||||
|
// module's request builders. Dynamic selections are checked at execution.
|
||||||
|
Prompts []string
|
||||||
ArtifactKind contracts.ArtifactKind
|
ArtifactKind contracts.ArtifactKind
|
||||||
Provides []string
|
Provides []string
|
||||||
Requires []string
|
Requires []string
|
||||||
@@ -33,12 +35,11 @@ type ModuleSpec struct {
|
|||||||
|
|
||||||
func normalizeModuleSpec(spec ModuleSpec) ModuleSpec {
|
func normalizeModuleSpec(spec ModuleSpec) ModuleSpec {
|
||||||
executionClass := contracts.ExecutionClass(strings.TrimSpace(string(spec.ExecutionClass)))
|
executionClass := contracts.ExecutionClass(strings.TrimSpace(string(spec.ExecutionClass)))
|
||||||
correctionProtocol := contracts.CorrectionProtocol(strings.TrimSpace(string(spec.CorrectionProtocol)))
|
|
||||||
return ModuleSpec{
|
return ModuleSpec{
|
||||||
Key: strings.TrimSpace(spec.Key),
|
Key: strings.TrimSpace(spec.Key),
|
||||||
Stage: spec.Stage,
|
Stage: spec.Stage,
|
||||||
ExecutionClass: executionClass,
|
ExecutionClass: executionClass,
|
||||||
CorrectionProtocol: correctionProtocol,
|
Prompts: normalizeStringSet(spec.Prompts),
|
||||||
ArtifactKind: normalizeArtifactKind(spec.ArtifactKind),
|
ArtifactKind: normalizeArtifactKind(spec.ArtifactKind),
|
||||||
Provides: normalizeCapabilities(spec.Provides),
|
Provides: normalizeCapabilities(spec.Provides),
|
||||||
Requires: normalizeCapabilities(spec.Requires),
|
Requires: normalizeCapabilities(spec.Requires),
|
||||||
@@ -76,7 +77,7 @@ func cloneModuleSpec(spec ModuleSpec) ModuleSpec {
|
|||||||
Key: spec.Key,
|
Key: spec.Key,
|
||||||
Stage: spec.Stage,
|
Stage: spec.Stage,
|
||||||
ExecutionClass: spec.ExecutionClass,
|
ExecutionClass: spec.ExecutionClass,
|
||||||
CorrectionProtocol: spec.CorrectionProtocol,
|
Prompts: append([]string(nil), spec.Prompts...),
|
||||||
ArtifactKind: spec.ArtifactKind,
|
ArtifactKind: spec.ArtifactKind,
|
||||||
Provides: append([]string(nil), spec.Provides...),
|
Provides: append([]string(nil), spec.Provides...),
|
||||||
Requires: append([]string(nil), spec.Requires...),
|
Requires: append([]string(nil), spec.Requires...),
|
||||||
@@ -97,19 +98,6 @@ func validateModuleSpec(kind string, expectedStage ModuleStage, spec ModuleSpec)
|
|||||||
if spec.ExecutionClass != contracts.ExecutionClassDeterministic && spec.ExecutionClass != contracts.ExecutionClassLLMBacked {
|
if spec.ExecutionClass != contracts.ExecutionClassDeterministic && spec.ExecutionClass != contracts.ExecutionClassLLMBacked {
|
||||||
return fmt.Errorf("%s %q has unsupported execution class %q", kind, spec.Key, spec.ExecutionClass)
|
return fmt.Errorf("%s %q has unsupported execution class %q", kind, spec.Key, spec.ExecutionClass)
|
||||||
}
|
}
|
||||||
if spec.CorrectionProtocol != "" {
|
|
||||||
if err := spec.CorrectionProtocol.Validate(); err != nil {
|
|
||||||
return fmt.Errorf("%s %q correction protocol: %w", kind, spec.Key, err)
|
|
||||||
}
|
|
||||||
if spec.ExecutionClass != contracts.ExecutionClassLLMBacked {
|
|
||||||
return fmt.Errorf("%s %q correction protocol requires an LLM-backed execution class", kind, spec.Key)
|
|
||||||
}
|
|
||||||
switch spec.Stage {
|
|
||||||
case StageChunk, StageExtract, StageMerge, StageNormalize:
|
|
||||||
default:
|
|
||||||
return fmt.Errorf("%s %q correction protocol is not supported for %q stage", kind, spec.Key, spec.Stage)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if spec.ArtifactKind != "" && spec.Stage != StageExtract && spec.Stage != StageMerge && spec.Stage != StageNormalize {
|
if spec.ArtifactKind != "" && spec.Stage != StageExtract && spec.Stage != StageMerge && spec.Stage != StageNormalize {
|
||||||
return fmt.Errorf("%s %q must not declare an artifact kind", kind, spec.Key)
|
return fmt.Errorf("%s %q must not declare an artifact kind", kind, spec.Key)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -32,67 +32,41 @@ func TestValidateModuleSpecRequiresSupportedExecutionClass(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCloneModuleSpecPreservesCorrectionProtocol(t *testing.T) {
|
func TestModuleSpecRetainsDetachedStaticPromptDependencies(t *testing.T) {
|
||||||
spec := normalizeModuleSpec(ModuleSpec{
|
const initial = "notarius/test.initial"
|
||||||
Key: " module ",
|
const correction = "notarius/test.correction"
|
||||||
Stage: StageChunk,
|
prompts := []string{correction, initial}
|
||||||
ExecutionClass: contracts.ExecutionClassLLMBacked,
|
registry := NewExtractorRegistry()
|
||||||
CorrectionProtocol: " single_response_v1 ",
|
if err := RegisterExtractor(registry, ModuleSpec{Key: "producer", Stage: StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: "test/value", Prompts: prompts}, func() (contracts.Extractor[collaborationValue], error) { return &collaborationProducer{}, nil }); err != nil {
|
||||||
})
|
t.Fatal(err)
|
||||||
if spec.CorrectionProtocol != contracts.CorrectionProtocolSingleResponseV1 {
|
|
||||||
t.Fatalf("normalized CorrectionProtocol = %q, want %q", spec.CorrectionProtocol, contracts.CorrectionProtocolSingleResponseV1)
|
|
||||||
}
|
}
|
||||||
cloned := cloneModuleSpec(spec)
|
prompts[0] = "changed"
|
||||||
if !reflect.DeepEqual(cloned, spec) {
|
spec, ok := registry.Spec("producer")
|
||||||
t.Fatalf("cloneModuleSpec() = %#v, want %#v", cloned, spec)
|
if !ok || !reflect.DeepEqual(spec.Prompts, []string{correction, initial}) {
|
||||||
|
t.Fatalf("static dependencies = %#v", spec.Prompts)
|
||||||
|
}
|
||||||
|
spec.Prompts[0] = "also changed"
|
||||||
|
again, _ := registry.Spec("producer")
|
||||||
|
if !reflect.DeepEqual(again.Prompts, []string{correction, initial}) {
|
||||||
|
t.Fatalf("registry declarations mutated: %#v", again.Prompts)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestValidateModuleSpecCorrectionProtocolEligibility(t *testing.T) {
|
func TestValidatorSpecRetainsDetachedPromptDependencies(t *testing.T) {
|
||||||
tests := []struct {
|
prompts := []string{"notarius/assessment"}
|
||||||
name string
|
registry := NewValidatorRegistry()
|
||||||
stage ModuleStage
|
if err := RegisterTypedValidator(registry, "test/value", ValidatorSpec{Key: "assessment", ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: prompts}, func() (contracts.TypedValidator[collaborationValue], error) { return collaborationValidator{}, nil }); err != nil {
|
||||||
class contracts.ExecutionClass
|
t.Fatal(err)
|
||||||
protocol contracts.CorrectionProtocol
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
{name: "LLM chunk", stage: StageChunk, class: contracts.ExecutionClassLLMBacked, protocol: contracts.CorrectionProtocolSingleResponseV1},
|
|
||||||
{name: "LLM extract", stage: StageExtract, class: contracts.ExecutionClassLLMBacked, protocol: contracts.CorrectionProtocolSingleResponseV1},
|
|
||||||
{name: "LLM merge", stage: StageMerge, class: contracts.ExecutionClassLLMBacked, protocol: contracts.CorrectionProtocolSingleResponseV1},
|
|
||||||
{name: "LLM normalize", stage: StageNormalize, class: contracts.ExecutionClassLLMBacked, protocol: contracts.CorrectionProtocolSingleResponseV1},
|
|
||||||
{name: "deterministic chunk", stage: StageChunk, class: contracts.ExecutionClassDeterministic, protocol: contracts.CorrectionProtocolSingleResponseV1, want: "LLM-backed"},
|
|
||||||
{name: "input", stage: StageInput, class: contracts.ExecutionClassLLMBacked, protocol: contracts.CorrectionProtocolSingleResponseV1, want: "not supported"},
|
|
||||||
{name: "output", stage: StageOutput, class: contracts.ExecutionClassLLMBacked, protocol: contracts.CorrectionProtocolSingleResponseV1, want: "not supported"},
|
|
||||||
{name: "unknown protocol", stage: StageChunk, class: contracts.ExecutionClassLLMBacked, protocol: "unsupported", want: "unsupported correction protocol"},
|
|
||||||
}
|
}
|
||||||
|
prompts[0] = "changed"
|
||||||
for _, test := range tests {
|
spec, ok := registry.Spec("assessment")
|
||||||
t.Run(test.name, func(t *testing.T) {
|
if !ok || !reflect.DeepEqual(spec.Prompts, []string{"notarius/assessment"}) {
|
||||||
spec := normalizeModuleSpec(ModuleSpec{
|
t.Fatalf("validator dependencies = %#v", spec.Prompts)
|
||||||
Key: "module",
|
|
||||||
Stage: test.stage,
|
|
||||||
ExecutionClass: test.class,
|
|
||||||
CorrectionProtocol: test.protocol,
|
|
||||||
})
|
|
||||||
err := validateModuleSpec("module", test.stage, spec)
|
|
||||||
if test.want == "" && err != nil {
|
|
||||||
t.Fatalf("validateModuleSpec() error = %v, want nil", err)
|
|
||||||
}
|
}
|
||||||
if test.want != "" && (err == nil || !strings.Contains(err.Error(), test.want)) {
|
spec.Prompts[0] = "also changed"
|
||||||
t.Fatalf("validateModuleSpec() error = %v, want %q", err, test.want)
|
again, _ := registry.Spec("assessment")
|
||||||
}
|
if !reflect.DeepEqual(again.Prompts, []string{"notarius/assessment"}) {
|
||||||
})
|
t.Fatalf("validator declarations mutated: %#v", again.Prompts)
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNormalizeValidatorSpecRejectsCorrectionProtocol(t *testing.T) {
|
|
||||||
_, err := normalizeValidatorSpec(ValidatorSpec{
|
|
||||||
Key: "validator",
|
|
||||||
ExecutionClass: contracts.ExecutionClassLLMBacked,
|
|
||||||
CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1,
|
|
||||||
})
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "correction protocol") {
|
|
||||||
t.Fatalf("normalizeValidatorSpec() error = %v, want correction protocol error", err)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -76,36 +76,19 @@ func RegisterNormalizerBuilder[T any](registry *NormalizerRegistry, spec ModuleS
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return erasedTypedResult{}, err
|
return erasedTypedResult{}, err
|
||||||
}
|
}
|
||||||
correction, err := contracts.CloneSemanticCorrection(request.Correction)
|
result, err := normalizer.Normalize(ctx, contracts.TypedNormalizeRequest[T]{
|
||||||
if err != nil {
|
Source: request.Source, LaneID: request.LaneID, MergeOutput: contracts.MergeArtifact[T]{LaneID: request.MergeOutput.LaneID, MergerKey: request.MergeOutput.MergerKey, SourceID: request.MergeOutput.SourceID, Value: value},
|
||||||
return erasedTypedResult{}, fmt.Errorf("clone normalize correction: %w", err)
|
SourceInput: request.SourceInput, SessionID: request.SessionID, References: request.References, LLMProfile: request.LLMProfile, Metadata: request.Metadata, Operation: specializeProducerOperation[T](request.Operation),
|
||||||
}
|
})
|
||||||
result, err := normalizer.Normalize(ctx, contracts.TypedNormalizeRequest[T]{Source: request.Source, LaneID: request.LaneID, MergeOutput: contracts.MergeArtifact[T]{LaneID: request.MergeOutput.LaneID, MergerKey: request.MergeOutput.MergerKey, SourceID: request.MergeOutput.SourceID, Value: value}, SourceInput: request.SourceInput, SessionID: request.SessionID, References: request.References, LLMProfile: request.LLMProfile, StructuredOutputRepairAttempts: request.StructuredOutputRepairAttempts, Correction: correction, Metadata: request.Metadata})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return erasedTypedResult{}, err
|
return erasedTypedResult{}, err
|
||||||
}
|
}
|
||||||
candidate, err := contracts.CloneModelCandidate(result.ModelCandidate)
|
return erasedTypedResult{Value: result.Value, Diagnostics: contracts.CloneProducerDiagnostics(result.Diagnostics), Outcome: result.Outcome}, nil
|
||||||
if err != nil {
|
|
||||||
return erasedTypedResult{}, fmt.Errorf("clone normalize model candidate: %w", err)
|
|
||||||
}
|
|
||||||
return erasedTypedResult{Value: result.Value, Diagnostics: contracts.CloneProducerDiagnostics(result.Diagnostics), Retry: cloneNormalizeRetry(result.Retry), ModelCandidate: candidate}, nil
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func cloneNormalizeRetry(retry *contracts.NormalizeRetry) *contracts.NormalizeRetry {
|
|
||||||
if retry == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return &contracts.NormalizeRetry{
|
|
||||||
ReasonCode: retry.ReasonCode,
|
|
||||||
Message: retry.Message,
|
|
||||||
CorrectionGuidance: retry.CorrectionGuidance,
|
|
||||||
FallbackDiagnostics: contracts.CloneProducerDiagnostics(retry.FallbackDiagnostics),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *NormalizerRegistry) validateOptions(key string, kind contracts.ArtifactKind, options map[string]any) error {
|
func (r *NormalizerRegistry) validateOptions(key string, kind contracts.ArtifactKind, options map[string]any) error {
|
||||||
if r == nil {
|
if r == nil {
|
||||||
return fmt.Errorf("normalizer registry must not be nil")
|
return fmt.Errorf("normalizer registry must not be nil")
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ import (
|
|||||||
type PreparedPipeline struct {
|
type PreparedPipeline struct {
|
||||||
Input ModuleBinding
|
Input ModuleBinding
|
||||||
Chunk ModuleBinding
|
Chunk ModuleBinding
|
||||||
ChunkCorrectionProtocol contracts.CorrectionProtocol
|
ChunkPrompts []string
|
||||||
Steps []PreparedPipelineStep
|
Steps []PreparedPipelineStep
|
||||||
Output ModuleBinding
|
Output ModuleBinding
|
||||||
|
|
||||||
@@ -38,9 +38,9 @@ type PreparedPipelineStep struct {
|
|||||||
|
|
||||||
type PreparedArtifactLane struct {
|
type PreparedArtifactLane struct {
|
||||||
Resolved ResolvedArtifactLane
|
Resolved ResolvedArtifactLane
|
||||||
ExtractCorrectionProtocol contracts.CorrectionProtocol
|
ExtractPrompts []string
|
||||||
MergeCorrectionProtocol contracts.CorrectionProtocol
|
MergePrompts []string
|
||||||
NormalizeCorrectionProtocol contracts.CorrectionProtocol
|
NormalizePrompts []string
|
||||||
}
|
}
|
||||||
|
|
||||||
type preparedLaneExecutor struct {
|
type preparedLaneExecutor struct {
|
||||||
@@ -92,9 +92,6 @@ func Prepare(resolved ResolvedPipeline, registries Registries, deps ModuleDepend
|
|||||||
if err := validateResolvedPipeline(resolved); err != nil {
|
if err := validateResolvedPipeline(resolved); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if err := validateCorrectionRetryCapabilities(resolved); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if err := validateRegistrySet(resolved, registries); err != nil {
|
if err := validateRegistrySet(resolved, registries); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -102,7 +99,7 @@ func Prepare(resolved ResolvedPipeline, registries Registries, deps ModuleDepend
|
|||||||
prepared := &PreparedPipeline{
|
prepared := &PreparedPipeline{
|
||||||
Input: cloneModuleBinding(stable.Input),
|
Input: cloneModuleBinding(stable.Input),
|
||||||
Chunk: cloneModuleBinding(stable.Chunk),
|
Chunk: cloneModuleBinding(stable.Chunk),
|
||||||
ChunkCorrectionProtocol: stable.ChunkCorrectionProtocol,
|
ChunkPrompts: append([]string(nil), stable.ChunkPrompts...),
|
||||||
Output: cloneModuleBinding(stable.Output),
|
Output: cloneModuleBinding(stable.Output),
|
||||||
resolved: stable,
|
resolved: stable,
|
||||||
dependencies: deps,
|
dependencies: deps,
|
||||||
@@ -142,9 +139,9 @@ func Prepare(resolved ResolvedPipeline, registries Registries, deps ModuleDepend
|
|||||||
}
|
}
|
||||||
preparedStep.ArtifactLanes = append(preparedStep.ArtifactLanes, PreparedArtifactLane{
|
preparedStep.ArtifactLanes = append(preparedStep.ArtifactLanes, PreparedArtifactLane{
|
||||||
Resolved: cloneResolvedArtifactLane(lane),
|
Resolved: cloneResolvedArtifactLane(lane),
|
||||||
ExtractCorrectionProtocol: lane.ExtractCorrectionProtocol,
|
ExtractPrompts: append([]string(nil), lane.ExtractPrompts...),
|
||||||
MergeCorrectionProtocol: lane.MergeCorrectionProtocol,
|
MergePrompts: append([]string(nil), lane.MergePrompts...),
|
||||||
NormalizeCorrectionProtocol: lane.NormalizeCorrectionProtocol,
|
NormalizePrompts: append([]string(nil), lane.NormalizePrompts...),
|
||||||
})
|
})
|
||||||
preparedStep.lanes = append(preparedStep.lanes, executor)
|
preparedStep.lanes = append(preparedStep.lanes, executor)
|
||||||
}
|
}
|
||||||
@@ -213,40 +210,6 @@ func prepareEvidencePlan(resolved ResolvedPipeline, registries Registries, outpu
|
|||||||
return plan, nil
|
return plan, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateCorrectionRetryCapabilities(pipeline ResolvedPipeline) error {
|
|
||||||
validate := func(stage ModuleStage, laneID string, binding ModuleBinding, executionClass contracts.ExecutionClass, protocol contracts.CorrectionProtocol) error {
|
|
||||||
if executionClass != contracts.ExecutionClassLLMBacked || binding.Retries == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
chain := resolvedValidatorChain(stage, laneID, binding.Module, pipeline.ValidatorChains)
|
|
||||||
if len(chain.Validators) == 0 || protocol == contracts.CorrectionProtocolSingleResponseV1 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if laneID == "" {
|
|
||||||
return fmt.Errorf("pipeline %q %s module %q configures validators and retries but does not declare correction protocol %q", pipeline.ID, stage, binding.Module, contracts.CorrectionProtocolSingleResponseV1)
|
|
||||||
}
|
|
||||||
return fmt.Errorf("pipeline %q lane %q %s module %q configures validators and retries but does not declare correction protocol %q", pipeline.ID, laneID, stage, binding.Module, contracts.CorrectionProtocolSingleResponseV1)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := validate(StageChunk, "", pipeline.Chunk, pipeline.ChunkExecutionClass, pipeline.ChunkCorrectionProtocol); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
for _, step := range pipeline.Steps {
|
|
||||||
for _, lane := range step.ArtifactLanes {
|
|
||||||
if err := validate(StageExtract, lane.ID, lane.Extract, lane.ExtractExecutionClass, lane.ExtractCorrectionProtocol); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := validate(StageMerge, lane.ID, lane.Merge, lane.MergeExecutionClass, lane.MergeCorrectionProtocol); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := validate(StageNormalize, lane.ID, lane.Normalize, lane.NormalizeExecutionClass, lane.NormalizeCorrectionProtocol); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func prepareLane(pipeline ResolvedPipeline, lane ResolvedArtifactLane, registries Registries, deps ModuleDependencies) (preparedLaneExecutor, error) {
|
func prepareLane(pipeline ResolvedPipeline, lane ResolvedArtifactLane, registries Registries, deps ModuleDependencies) (preparedLaneExecutor, error) {
|
||||||
executor := preparedLaneExecutor{resolved: cloneResolvedArtifactLane(lane)}
|
executor := preparedLaneExecutor{resolved: cloneResolvedArtifactLane(lane)}
|
||||||
request := func(binding ModuleBinding, references contracts.ReferenceSet) BuildRequest {
|
request := func(binding ModuleBinding, references contracts.ReferenceSet) BuildRequest {
|
||||||
@@ -454,7 +417,7 @@ func validateRegistrySet(resolved ResolvedPipeline, registries Registries) error
|
|||||||
|
|
||||||
func cloneResolvedPipeline(in ResolvedPipeline) ResolvedPipeline {
|
func cloneResolvedPipeline(in ResolvedPipeline) ResolvedPipeline {
|
||||||
out := in
|
out := in
|
||||||
out.ConfiguredValidationPolicy = cloneValidationPolicyOverride(in.ConfiguredValidationPolicy)
|
out.ChunkPrompts = append([]string(nil), in.ChunkPrompts...)
|
||||||
out.Input = cloneModuleBinding(in.Input)
|
out.Input = cloneModuleBinding(in.Input)
|
||||||
out.Chunk = cloneModuleBinding(in.Chunk)
|
out.Chunk = cloneModuleBinding(in.Chunk)
|
||||||
out.Output = cloneModuleBinding(in.Output)
|
out.Output = cloneModuleBinding(in.Output)
|
||||||
@@ -477,6 +440,9 @@ func cloneResolvedPipeline(in ResolvedPipeline) ResolvedPipeline {
|
|||||||
|
|
||||||
func cloneResolvedArtifactLane(in ResolvedArtifactLane) ResolvedArtifactLane {
|
func cloneResolvedArtifactLane(in ResolvedArtifactLane) ResolvedArtifactLane {
|
||||||
out := in
|
out := in
|
||||||
|
out.ExtractPrompts = append([]string(nil), in.ExtractPrompts...)
|
||||||
|
out.MergePrompts = append([]string(nil), in.MergePrompts...)
|
||||||
|
out.NormalizePrompts = append([]string(nil), in.NormalizePrompts...)
|
||||||
out.Extract = cloneModuleBinding(in.Extract)
|
out.Extract = cloneModuleBinding(in.Extract)
|
||||||
out.Merge = cloneModuleBinding(in.Merge)
|
out.Merge = cloneModuleBinding(in.Merge)
|
||||||
out.Normalize = cloneModuleBinding(in.Normalize)
|
out.Normalize = cloneModuleBinding(in.Normalize)
|
||||||
|
|||||||
@@ -21,9 +21,9 @@ const (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type ModuleBinding struct {
|
type ModuleBinding struct {
|
||||||
Mode *ExecutionMode `json:"mode,omitempty"`
|
Mode *contracts.ExecutionMode `json:"mode,omitempty"`
|
||||||
MaxAdditionalTurns *int `json:"max_additional_turns,omitempty"`
|
MaxAdditionalTurns *int `json:"max_additional_turns,omitempty"`
|
||||||
Cache *CacheMode `json:"cache,omitempty"`
|
Cache *contracts.CacheMode `json:"cache,omitempty"`
|
||||||
Module string `json:"module"`
|
Module string `json:"module"`
|
||||||
LLMProfile string `json:"llm_profile,omitempty"`
|
LLMProfile string `json:"llm_profile,omitempty"`
|
||||||
Options map[string]any `json:"options,omitempty"`
|
Options map[string]any `json:"options,omitempty"`
|
||||||
@@ -82,9 +82,9 @@ type ValidatorOverride struct {
|
|||||||
|
|
||||||
func (binding ModuleBinding) MarshalJSON() ([]byte, error) {
|
func (binding ModuleBinding) MarshalJSON() ([]byte, error) {
|
||||||
type moduleBindingJSON struct {
|
type moduleBindingJSON struct {
|
||||||
Mode *ExecutionMode `json:"mode,omitempty"`
|
Mode *contracts.ExecutionMode `json:"mode,omitempty"`
|
||||||
MaxAdditionalTurns *int `json:"max_additional_turns,omitempty"`
|
MaxAdditionalTurns *int `json:"max_additional_turns,omitempty"`
|
||||||
Cache *CacheMode `json:"cache,omitempty"`
|
Cache *contracts.CacheMode `json:"cache,omitempty"`
|
||||||
Module string `json:"module"`
|
Module string `json:"module"`
|
||||||
LLMProfile string `json:"llm_profile,omitempty"`
|
LLMProfile string `json:"llm_profile,omitempty"`
|
||||||
Options map[string]any `json:"options,omitempty"`
|
Options map[string]any `json:"options,omitempty"`
|
||||||
@@ -112,9 +112,9 @@ func (binding *ModuleBinding) UnmarshalJSON(data []byte) error {
|
|||||||
return fmt.Errorf("module binding must not be nil")
|
return fmt.Errorf("module binding must not be nil")
|
||||||
}
|
}
|
||||||
type moduleBindingJSON struct {
|
type moduleBindingJSON struct {
|
||||||
Mode *ExecutionMode `json:"mode,omitempty"`
|
Mode *contracts.ExecutionMode `json:"mode,omitempty"`
|
||||||
MaxAdditionalTurns *int `json:"max_additional_turns,omitempty"`
|
MaxAdditionalTurns *int `json:"max_additional_turns,omitempty"`
|
||||||
Cache *CacheMode `json:"cache,omitempty"`
|
Cache *contracts.CacheMode `json:"cache,omitempty"`
|
||||||
Module string `json:"module"`
|
Module string `json:"module"`
|
||||||
LLMProfile string `json:"llm_profile,omitempty"`
|
LLMProfile string `json:"llm_profile,omitempty"`
|
||||||
Options map[string]any `json:"options,omitempty"`
|
Options map[string]any `json:"options,omitempty"`
|
||||||
@@ -160,9 +160,9 @@ type PipelineStepProfile struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type PipelineProfile struct {
|
type PipelineProfile struct {
|
||||||
Mode *ExecutionMode `json:"mode,omitempty"`
|
Mode *contracts.ExecutionMode `json:"mode,omitempty"`
|
||||||
MaxAdditionalTurns *int `json:"max_additional_turns,omitempty"`
|
MaxAdditionalTurns *int `json:"max_additional_turns,omitempty"`
|
||||||
Cache *CacheMode `json:"cache,omitempty"`
|
Cache *contracts.CacheMode `json:"cache,omitempty"`
|
||||||
ID string `json:"id"`
|
ID string `json:"id"`
|
||||||
LLMProfile string `json:"llm_profile,omitempty"`
|
LLMProfile string `json:"llm_profile,omitempty"`
|
||||||
Input ModuleBinding `json:"input"`
|
Input ModuleBinding `json:"input"`
|
||||||
@@ -174,7 +174,7 @@ type PipelineProfile struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type ResolveOptions struct {
|
type ResolveOptions struct {
|
||||||
CacheOverride *CacheMode
|
CacheOverride *contracts.CacheMode
|
||||||
Only []string
|
Only []string
|
||||||
LLMProfileOverride string
|
LLMProfileOverride string
|
||||||
ReferenceOverrides []ReferenceBinding
|
ReferenceOverrides []ReferenceBinding
|
||||||
@@ -206,9 +206,9 @@ type ResolvedReferenceTarget struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type ResolvedArtifactLane struct {
|
type ResolvedArtifactLane struct {
|
||||||
ExtractControls ProducerControls
|
ExtractControls contracts.ProducerControls
|
||||||
MergeControls ProducerControls
|
MergeControls contracts.ProducerControls
|
||||||
NormalizeControls ProducerControls
|
NormalizeControls contracts.ProducerControls
|
||||||
StepID string
|
StepID string
|
||||||
ID string
|
ID string
|
||||||
ArtifactKind contracts.ArtifactKind `json:"artifact_kind,omitempty"`
|
ArtifactKind contracts.ArtifactKind `json:"artifact_kind,omitempty"`
|
||||||
@@ -218,13 +218,13 @@ type ResolvedArtifactLane struct {
|
|||||||
ArtifactSchemaDigest string `json:"artifact_schema_digest,omitempty"`
|
ArtifactSchemaDigest string `json:"artifact_schema_digest,omitempty"`
|
||||||
Extract ModuleBinding
|
Extract ModuleBinding
|
||||||
ExtractExecutionClass contracts.ExecutionClass `json:"extract_execution_class"`
|
ExtractExecutionClass contracts.ExecutionClass `json:"extract_execution_class"`
|
||||||
ExtractCorrectionProtocol contracts.CorrectionProtocol `json:"extract_correction_protocol,omitempty"`
|
ExtractPrompts []string
|
||||||
Merge ModuleBinding
|
Merge ModuleBinding
|
||||||
MergeExecutionClass contracts.ExecutionClass `json:"merge_execution_class"`
|
MergeExecutionClass contracts.ExecutionClass `json:"merge_execution_class"`
|
||||||
MergeCorrectionProtocol contracts.CorrectionProtocol `json:"merge_correction_protocol,omitempty"`
|
MergePrompts []string
|
||||||
Normalize ModuleBinding
|
Normalize ModuleBinding
|
||||||
NormalizeExecutionClass contracts.ExecutionClass `json:"normalize_execution_class"`
|
NormalizeExecutionClass contracts.ExecutionClass `json:"normalize_execution_class"`
|
||||||
NormalizeCorrectionProtocol contracts.CorrectionProtocol `json:"normalize_correction_protocol,omitempty"`
|
NormalizePrompts []string
|
||||||
Validators []ModuleBinding
|
Validators []ModuleBinding
|
||||||
ExtractReferences ResolvedReferenceTarget `json:"extract_references"`
|
ExtractReferences ResolvedReferenceTarget `json:"extract_references"`
|
||||||
MergeReferences ResolvedReferenceTarget `json:"merge_references"`
|
MergeReferences ResolvedReferenceTarget `json:"merge_references"`
|
||||||
@@ -244,6 +244,7 @@ type ResolvedValidatorChain struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type ResolvedValidator struct {
|
type ResolvedValidator struct {
|
||||||
|
Prompts []string `json:"prompts,omitempty"`
|
||||||
Binding ModuleBinding `json:"binding"`
|
Binding ModuleBinding `json:"binding"`
|
||||||
ExecutionClass contracts.ExecutionClass `json:"execution_class"`
|
ExecutionClass contracts.ExecutionClass `json:"execution_class"`
|
||||||
Target ValidatorTarget `json:"target,omitempty"`
|
Target ValidatorTarget `json:"target,omitempty"`
|
||||||
@@ -251,14 +252,14 @@ type ResolvedValidator struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type ResolvedPipeline struct {
|
type ResolvedPipeline struct {
|
||||||
ChunkControls ProducerControls
|
ChunkControls contracts.ProducerControls
|
||||||
ID string
|
ID string
|
||||||
Digest string
|
Digest string
|
||||||
Input ModuleBinding
|
Input ModuleBinding
|
||||||
InputExecutionClass contracts.ExecutionClass `json:"input_execution_class"`
|
InputExecutionClass contracts.ExecutionClass `json:"input_execution_class"`
|
||||||
Chunk ModuleBinding
|
Chunk ModuleBinding
|
||||||
ChunkExecutionClass contracts.ExecutionClass `json:"chunk_execution_class"`
|
ChunkExecutionClass contracts.ExecutionClass `json:"chunk_execution_class"`
|
||||||
ChunkCorrectionProtocol contracts.CorrectionProtocol `json:"chunk_correction_protocol,omitempty"`
|
ChunkPrompts []string
|
||||||
ChunkReferences ResolvedReferenceTarget `json:"chunk_references"`
|
ChunkReferences ResolvedReferenceTarget `json:"chunk_references"`
|
||||||
Steps []ResolvedPipelineStep
|
Steps []ResolvedPipelineStep
|
||||||
ValidatorChains []ResolvedValidatorChain `json:"validator_chains"`
|
ValidatorChains []ResolvedValidatorChain `json:"validator_chains"`
|
||||||
@@ -435,7 +436,7 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
|
|||||||
InputExecutionClass: inputModuleSpec.ExecutionClass,
|
InputExecutionClass: inputModuleSpec.ExecutionClass,
|
||||||
Chunk: chunk,
|
Chunk: chunk,
|
||||||
ChunkExecutionClass: chunkSpec.ExecutionClass,
|
ChunkExecutionClass: chunkSpec.ExecutionClass,
|
||||||
ChunkCorrectionProtocol: chunkSpec.CorrectionProtocol,
|
ChunkPrompts: append([]string(nil), chunkSpec.Prompts...),
|
||||||
ChunkReferences: referenceTarget(StageChunk, "", chunk.Module, chunkReferences),
|
ChunkReferences: referenceTarget(StageChunk, "", chunk.Module, chunkReferences),
|
||||||
Output: output,
|
Output: output,
|
||||||
}
|
}
|
||||||
@@ -617,7 +618,7 @@ func resolveArtifactLane(
|
|||||||
lane.ExtractReferences = referenceTarget(StageExtract, laneID, lane.Extract.Module, references)
|
lane.ExtractReferences = referenceTarget(StageExtract, laneID, lane.Extract.Module, references)
|
||||||
lane.ExtractReferences.StepID = strings.TrimSpace(stepID)
|
lane.ExtractReferences.StepID = strings.TrimSpace(stepID)
|
||||||
lane.ExtractExecutionClass = extractSpec.ExecutionClass
|
lane.ExtractExecutionClass = extractSpec.ExecutionClass
|
||||||
lane.ExtractCorrectionProtocol = extractSpec.CorrectionProtocol
|
lane.ExtractPrompts = append([]string(nil), extractSpec.Prompts...)
|
||||||
capabilities.add(extractSpec.Provides...)
|
capabilities.add(extractSpec.Provides...)
|
||||||
|
|
||||||
mergeSpec, err := mergerSpecForArtifact(catalog, lane.Merge.Module, lane.ArtifactKind, artifactType)
|
mergeSpec, err := mergerSpecForArtifact(catalog, lane.Merge.Module, lane.ArtifactKind, artifactType)
|
||||||
@@ -644,7 +645,7 @@ func resolveArtifactLane(
|
|||||||
lane.MergeReferences = referenceTarget(StageMerge, laneID, lane.Merge.Module, mergeReferences)
|
lane.MergeReferences = referenceTarget(StageMerge, laneID, lane.Merge.Module, mergeReferences)
|
||||||
lane.MergeReferences.StepID = strings.TrimSpace(stepID)
|
lane.MergeReferences.StepID = strings.TrimSpace(stepID)
|
||||||
lane.MergeExecutionClass = mergeSpec.ExecutionClass
|
lane.MergeExecutionClass = mergeSpec.ExecutionClass
|
||||||
lane.MergeCorrectionProtocol = mergeSpec.CorrectionProtocol
|
lane.MergePrompts = append([]string(nil), mergeSpec.Prompts...)
|
||||||
capabilities.add(mergeSpec.Provides...)
|
capabilities.add(mergeSpec.Provides...)
|
||||||
|
|
||||||
normalizeSpec, err := normalizerSpecForArtifact(catalog, lane.Normalize.Module, lane.ArtifactKind, artifactType)
|
normalizeSpec, err := normalizerSpecForArtifact(catalog, lane.Normalize.Module, lane.ArtifactKind, artifactType)
|
||||||
@@ -671,7 +672,7 @@ func resolveArtifactLane(
|
|||||||
lane.NormalizeReferences = referenceTarget(StageNormalize, laneID, lane.Normalize.Module, normalizeReferences)
|
lane.NormalizeReferences = referenceTarget(StageNormalize, laneID, lane.Normalize.Module, normalizeReferences)
|
||||||
lane.NormalizeReferences.StepID = strings.TrimSpace(stepID)
|
lane.NormalizeReferences.StepID = strings.TrimSpace(stepID)
|
||||||
lane.NormalizeExecutionClass = normalizeSpec.ExecutionClass
|
lane.NormalizeExecutionClass = normalizeSpec.ExecutionClass
|
||||||
lane.NormalizeCorrectionProtocol = normalizeSpec.CorrectionProtocol
|
lane.NormalizePrompts = append([]string(nil), normalizeSpec.Prompts...)
|
||||||
capabilities.add(normalizeSpec.Provides...)
|
capabilities.add(normalizeSpec.Provides...)
|
||||||
|
|
||||||
if len(lane.Validators) > 0 {
|
if len(lane.Validators) > 0 {
|
||||||
@@ -936,6 +937,7 @@ func resolveValidatorChain(pipelineID string, laneID string, stage ModuleStage,
|
|||||||
chain.Validators = append(chain.Validators, ResolvedValidator{
|
chain.Validators = append(chain.Validators, ResolvedValidator{
|
||||||
Binding: cloneModuleBinding(validator),
|
Binding: cloneModuleBinding(validator),
|
||||||
ExecutionClass: spec.ExecutionClass,
|
ExecutionClass: spec.ExecutionClass,
|
||||||
|
Prompts: append([]string(nil), spec.Prompts...),
|
||||||
Target: target,
|
Target: target,
|
||||||
ArtifactKind: artifactKind,
|
ArtifactKind: artifactKind,
|
||||||
})
|
})
|
||||||
@@ -968,6 +970,7 @@ func cloneResolvedValidators(validators []ResolvedValidator) []ResolvedValidator
|
|||||||
out[i] = ResolvedValidator{
|
out[i] = ResolvedValidator{
|
||||||
Binding: cloneModuleBinding(validator.Binding),
|
Binding: cloneModuleBinding(validator.Binding),
|
||||||
ExecutionClass: validator.ExecutionClass,
|
ExecutionClass: validator.ExecutionClass,
|
||||||
|
Prompts: append([]string(nil), validator.Prompts...),
|
||||||
Target: validator.Target,
|
Target: validator.Target,
|
||||||
ArtifactKind: validator.ArtifactKind,
|
ArtifactKind: validator.ArtifactKind,
|
||||||
}
|
}
|
||||||
@@ -1517,13 +1520,13 @@ func selectedArtifactLanes(pipelineID string, artifacts map[string]ArtifactLaneP
|
|||||||
|
|
||||||
func resolvedPipelineDigest(resolved ResolvedPipeline) (string, error) {
|
func resolvedPipelineDigest(resolved ResolvedPipeline) (string, error) {
|
||||||
withoutDigest := struct {
|
withoutDigest := struct {
|
||||||
ChunkControls ProducerControls
|
ChunkControls contracts.ProducerControls
|
||||||
ID string
|
ID string
|
||||||
Input ModuleBinding
|
Input ModuleBinding
|
||||||
InputExecutionClass contracts.ExecutionClass
|
InputExecutionClass contracts.ExecutionClass
|
||||||
Chunk ModuleBinding
|
Chunk ModuleBinding
|
||||||
ChunkExecutionClass contracts.ExecutionClass
|
ChunkExecutionClass contracts.ExecutionClass
|
||||||
ChunkCorrectionProtocol contracts.CorrectionProtocol
|
ChunkPrompts []string
|
||||||
ChunkReferences ResolvedReferenceTarget
|
ChunkReferences ResolvedReferenceTarget
|
||||||
Steps []ResolvedPipelineStep
|
Steps []ResolvedPipelineStep
|
||||||
ValidatorChains []ResolvedValidatorChain
|
ValidatorChains []ResolvedValidatorChain
|
||||||
@@ -1536,7 +1539,7 @@ func resolvedPipelineDigest(resolved ResolvedPipeline) (string, error) {
|
|||||||
InputExecutionClass: resolved.InputExecutionClass,
|
InputExecutionClass: resolved.InputExecutionClass,
|
||||||
Chunk: resolved.Chunk,
|
Chunk: resolved.Chunk,
|
||||||
ChunkExecutionClass: resolved.ChunkExecutionClass,
|
ChunkExecutionClass: resolved.ChunkExecutionClass,
|
||||||
ChunkCorrectionProtocol: resolved.ChunkCorrectionProtocol,
|
ChunkPrompts: append([]string(nil), resolved.ChunkPrompts...),
|
||||||
ChunkReferences: resolved.ChunkReferences,
|
ChunkReferences: resolved.ChunkReferences,
|
||||||
Steps: resolved.Steps,
|
Steps: resolved.Steps,
|
||||||
ValidatorChains: resolved.ValidatorChains,
|
ValidatorChains: resolved.ValidatorChains,
|
||||||
|
|||||||
@@ -24,8 +24,7 @@ type erasedMergeArtifact struct {
|
|||||||
type erasedTypedResult struct {
|
type erasedTypedResult struct {
|
||||||
Value any
|
Value any
|
||||||
Diagnostics []contracts.ProducerDiagnostic
|
Diagnostics []contracts.ProducerDiagnostic
|
||||||
Retry *contracts.NormalizeRetry
|
Outcome contracts.ProducerOutcome
|
||||||
ModelCandidate *contracts.ModelCandidate
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type typedValidationTarget struct {
|
type typedValidationTarget struct {
|
||||||
@@ -39,7 +38,7 @@ type typedValidationTarget struct {
|
|||||||
sessionID string
|
sessionID string
|
||||||
references contracts.ReferenceSet
|
references contracts.ReferenceSet
|
||||||
llmProfile string
|
llmProfile string
|
||||||
structuredOutputRepairAttempts *int
|
execution contracts.ExecutionOptions
|
||||||
metadata map[string]any
|
metadata map[string]any
|
||||||
chunk *source.Chunk
|
chunk *source.Chunk
|
||||||
chunks []source.Chunk
|
chunks []source.Chunk
|
||||||
@@ -62,7 +61,18 @@ func exactTypedValue[T any](operation string, value any) (T, error) {
|
|||||||
return typed, nil
|
return typed, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
type typedExtractOperation func(context.Context, any, contracts.TypedExtractionRequest) (erasedTypedResult, error)
|
type typedExtractOperation func(context.Context, any, contracts.TypedExtractionRequest[any]) (erasedTypedResult, error)
|
||||||
type typedMergeOperation func(context.Context, any, contracts.TypedMergeRequest[any]) (erasedTypedResult, error)
|
type typedMergeOperation func(context.Context, any, contracts.TypedMergeRequest[any]) (erasedTypedResult, error)
|
||||||
type typedNormalizeOperation func(context.Context, any, contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error)
|
type typedNormalizeOperation func(context.Context, any, contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error)
|
||||||
type typedValidateOperation func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error)
|
type typedValidateOperation func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error)
|
||||||
|
|
||||||
|
// specializeProducerOperation keeps type erasure inside the registry boundary.
|
||||||
|
func specializeProducerOperation[T any](operation contracts.ProducerOperation[any]) contracts.ProducerOperation[T] {
|
||||||
|
result := contracts.ProducerOperation[T]{Controls: operation.Controls, Execution: operation.Execution, ReuseEligible: operation.ReuseEligible}
|
||||||
|
if operation.Assess != nil {
|
||||||
|
result.Assess = func(ctx context.Context, value T) (contracts.CandidateAssessment, error) {
|
||||||
|
return operation.Assess(ctx, value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,253 +1,191 @@
|
|||||||
package pipeline
|
package pipeline
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
|
"reflect"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
"go.maximumdirect.net/promptkit"
|
||||||
)
|
)
|
||||||
|
|
||||||
type correctionObservingNotesExtractor struct {
|
type collaborationValue struct{ Text string }
|
||||||
correction *contracts.SemanticCorrection
|
|
||||||
candidate *contracts.ModelCandidate
|
type collaborationProducer struct {
|
||||||
|
operation contracts.ProducerOperation[collaborationValue]
|
||||||
|
outcome contracts.ProducerOutcome
|
||||||
}
|
}
|
||||||
|
|
||||||
func (*correctionObservingNotesExtractor) Key() string { return "test/correction-observing-extract" }
|
func (*collaborationProducer) Key() string { return "producer" }
|
||||||
func (*correctionObservingNotesExtractor) ReferenceSlots() []contracts.ReferenceSlot {
|
func (*collaborationProducer) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||||
|
|
||||||
|
func (p *collaborationProducer) assess(ctx context.Context, value collaborationValue, operation contracts.ProducerOperation[collaborationValue]) error {
|
||||||
|
p.operation = operation
|
||||||
|
if operation.Assess != nil {
|
||||||
|
_, err := operation.Assess(ctx, value)
|
||||||
|
return err
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *correctionObservingNotesExtractor) Extract(_ context.Context, request contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[codecNotes], error) {
|
func (p *collaborationProducer) Extract(ctx context.Context, request contracts.TypedExtractionRequest[collaborationValue]) (contracts.TypedExtractionResult[collaborationValue], error) {
|
||||||
e.correction = request.Correction
|
value := collaborationValue{"candidate"}
|
||||||
candidate, err := contracts.NewModelCandidate([]byte(`{"items":["extract"]}`), contracts.CorrectionProtocolSingleResponseV1)
|
err := p.assess(ctx, value, request.Operation)
|
||||||
|
return contracts.TypedExtractionResult[collaborationValue]{Value: value, Outcome: p.outcome}, err
|
||||||
|
}
|
||||||
|
func (p *collaborationProducer) Merge(ctx context.Context, request contracts.TypedMergeRequest[collaborationValue]) (contracts.TypedMergeResult[collaborationValue], error) {
|
||||||
|
value := request.ExtractOutputs[0].Value
|
||||||
|
err := p.assess(ctx, value, request.Operation)
|
||||||
|
return contracts.TypedMergeResult[collaborationValue]{Value: value, Outcome: p.outcome}, err
|
||||||
|
}
|
||||||
|
func (p *collaborationProducer) Normalize(ctx context.Context, request contracts.TypedNormalizeRequest[collaborationValue]) (contracts.TypedNormalizeResult[collaborationValue], error) {
|
||||||
|
value := request.MergeOutput.Value
|
||||||
|
err := p.assess(ctx, value, request.Operation)
|
||||||
|
return contracts.TypedNormalizeResult[collaborationValue]{Value: value, Outcome: p.outcome}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
type collaborationValidator struct{ execution *contracts.ExecutionOptions }
|
||||||
|
|
||||||
|
func (collaborationValidator) Name() string { return "assessment" }
|
||||||
|
func (collaborationValidator) ExecutionClass() contracts.ExecutionClass {
|
||||||
|
return contracts.ExecutionClassLLMBacked
|
||||||
|
}
|
||||||
|
func (v collaborationValidator) Validate(_ context.Context, request contracts.TypedValidationRequest[collaborationValue]) (contracts.ValidationResult, error) {
|
||||||
|
if v.execution != nil {
|
||||||
|
*v.execution = request.Execution
|
||||||
|
}
|
||||||
|
return contracts.ValidationResult{Approved: true}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTypedRegistryCarriesOperationAssessmentAndOutcome(t *testing.T) {
|
||||||
|
for _, stage := range []ModuleStage{StageExtract, StageMerge, StageNormalize} {
|
||||||
|
t.Run(string(stage), func(t *testing.T) {
|
||||||
|
expected := collaborationValue{"candidate"}
|
||||||
|
calls := 0
|
||||||
|
zero := 0
|
||||||
|
native := &promptkit.ExecutionTargetOverride{}
|
||||||
|
operation := contracts.ProducerOperation[any]{
|
||||||
|
Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 0, Cache: contracts.CacheBypass},
|
||||||
|
Execution: contracts.ExecutionOptions{Execution: native, TransientRetries: &zero},
|
||||||
|
ReuseEligible: false,
|
||||||
|
Assess: func(_ context.Context, value any) (contracts.CandidateAssessment, error) {
|
||||||
|
calls++
|
||||||
|
if reflect.TypeOf(value) != reflect.TypeFor[collaborationValue]() || value != expected {
|
||||||
|
t.Fatalf("candidate = %#v", value)
|
||||||
|
}
|
||||||
|
return contracts.CandidateAssessment{}, nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
impl := &collaborationProducer{outcome: contracts.ProducerOutcome{Disposition: contracts.ProducerFallback, FallbackReason: "rejected", ReuseEligible: false}}
|
||||||
|
invoke := registerCollaborationProducer(t, stage, impl)
|
||||||
|
result, err := invoke(context.Background(), operation, expected)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return contracts.TypedExtractionResult[codecNotes]{}, err
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
e.candidate = candidate
|
if calls != 1 || result.Value != expected || result.Outcome != impl.outcome {
|
||||||
return contracts.TypedExtractionResult[codecNotes]{Value: codecNotes{Items: []string{"extract"}}, ModelCandidate: candidate}, nil
|
t.Fatalf("result = %#v, assessment calls = %d", result, calls)
|
||||||
}
|
|
||||||
|
|
||||||
type correctionObservingNotesMerger struct {
|
|
||||||
correction *contracts.SemanticCorrection
|
|
||||||
candidate *contracts.ModelCandidate
|
|
||||||
}
|
|
||||||
|
|
||||||
func (*correctionObservingNotesMerger) Key() string { return "test/correction-observing-merge" }
|
|
||||||
|
|
||||||
func (m *correctionObservingNotesMerger) Merge(_ context.Context, request contracts.TypedMergeRequest[codecNotes]) (contracts.TypedMergeResult[codecNotes], error) {
|
|
||||||
m.correction = request.Correction
|
|
||||||
candidate, err := contracts.NewModelCandidate([]byte(`{"items":["merge"]}`), contracts.CorrectionProtocolSingleResponseV1)
|
|
||||||
if err != nil {
|
|
||||||
return contracts.TypedMergeResult[codecNotes]{}, err
|
|
||||||
}
|
}
|
||||||
m.candidate = candidate
|
got := impl.operation
|
||||||
return contracts.TypedMergeResult[codecNotes]{Value: codecNotes{Items: []string{"merge"}}, ModelCandidate: candidate}, nil
|
if got.Controls != operation.Controls || got.ReuseEligible || got.Execution.Execution != native || got.Execution.TransientRetries == nil || *got.Execution.TransientRetries != 0 {
|
||||||
}
|
t.Fatalf("operation lost at type boundary: %#v", got)
|
||||||
|
|
||||||
type correctionObservingNotesNormalizer struct {
|
|
||||||
correction *contracts.SemanticCorrection
|
|
||||||
candidate *contracts.ModelCandidate
|
|
||||||
}
|
|
||||||
|
|
||||||
func (*correctionObservingNotesNormalizer) Key() string { return "test/correction-observing-normalize" }
|
|
||||||
func (*correctionObservingNotesNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n *correctionObservingNotesNormalizer) Normalize(_ context.Context, request contracts.TypedNormalizeRequest[codecNotes]) (contracts.TypedNormalizeResult[codecNotes], error) {
|
|
||||||
n.correction = request.Correction
|
|
||||||
candidate, err := contracts.NewModelCandidate([]byte(`{"items":["normalize"]}`), contracts.CorrectionProtocolSingleResponseV1)
|
|
||||||
if err != nil {
|
|
||||||
return contracts.TypedNormalizeResult[codecNotes]{}, err
|
|
||||||
}
|
}
|
||||||
n.candidate = candidate
|
sentinel := errors.New("assessment contract failure")
|
||||||
return contracts.TypedNormalizeResult[codecNotes]{Value: codecNotes{Items: []string{"normalize"}}, ModelCandidate: candidate}, nil
|
operation.Assess = func(context.Context, any) (contracts.CandidateAssessment, error) {
|
||||||
}
|
return contracts.CandidateAssessment{}, sentinel
|
||||||
|
|
||||||
type repairObservingNotesMerger struct {
|
|
||||||
attempts *int
|
|
||||||
}
|
|
||||||
|
|
||||||
func (*repairObservingNotesMerger) Key() string { return "test/repair-observing-merge" }
|
|
||||||
|
|
||||||
func (m *repairObservingNotesMerger) Merge(_ context.Context, request contracts.TypedMergeRequest[codecNotes]) (contracts.TypedMergeResult[codecNotes], error) {
|
|
||||||
m.attempts = request.StructuredOutputRepairAttempts
|
|
||||||
return contracts.TypedMergeResult[codecNotes]{Value: codecNotes{Items: []string{"merged"}}}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type repairObservingNotesNormalizer struct {
|
|
||||||
attempts *int
|
|
||||||
}
|
|
||||||
|
|
||||||
func (*repairObservingNotesNormalizer) Key() string { return "test/repair-observing-normalize" }
|
|
||||||
func (*repairObservingNotesNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n *repairObservingNotesNormalizer) Normalize(_ context.Context, request contracts.TypedNormalizeRequest[codecNotes]) (contracts.TypedNormalizeResult[codecNotes], error) {
|
|
||||||
n.attempts = request.StructuredOutputRepairAttempts
|
|
||||||
return contracts.TypedNormalizeResult[codecNotes]{Value: request.MergeOutput.Value}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTypedRegistryErasurePreservesStructuredOutputRepairAttempts(t *testing.T) {
|
|
||||||
const artifactKind contracts.ArtifactKind = "test/notes"
|
|
||||||
|
|
||||||
t.Run("merge", func(t *testing.T) {
|
|
||||||
implementation := &repairObservingNotesMerger{}
|
|
||||||
registry := NewMergerRegistry()
|
|
||||||
if err := RegisterMerger(registry, ModuleSpec{Key: implementation.Key(), Stage: StageMerge, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: artifactKind}, func() (contracts.Merger[codecNotes], error) {
|
|
||||||
return implementation, nil
|
|
||||||
}); err != nil {
|
|
||||||
t.Fatalf("RegisterMerger() error = %v", err)
|
|
||||||
}
|
}
|
||||||
entry, ok := registry.typedEntry(implementation.Key(), artifactKind)
|
if _, err := invoke(context.Background(), operation, expected); !errors.Is(err, sentinel) {
|
||||||
if !ok {
|
t.Fatalf("assessment error = %v", err)
|
||||||
t.Fatal("typed merger entry missing")
|
|
||||||
}
|
|
||||||
built, err := entry.builder(BuildRequest{})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("builder() error = %v", err)
|
|
||||||
}
|
|
||||||
attempts := 2
|
|
||||||
_, err = entry.merge(context.Background(), built, contracts.TypedMergeRequest[any]{
|
|
||||||
ExtractOutputs: []contracts.ExtractArtifact[any]{{Value: codecNotes{Items: []string{"extracted"}}}},
|
|
||||||
StructuredOutputRepairAttempts: &attempts,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("merge() error = %v", err)
|
|
||||||
}
|
|
||||||
if implementation.attempts == nil || *implementation.attempts != attempts {
|
|
||||||
t.Fatalf("repair attempts = %v, want %d", implementation.attempts, attempts)
|
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("normalize", func(t *testing.T) {
|
|
||||||
implementation := &repairObservingNotesNormalizer{}
|
|
||||||
registry := NewNormalizerRegistry()
|
|
||||||
if err := RegisterNormalizer(registry, ModuleSpec{Key: implementation.Key(), Stage: StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: artifactKind}, func() (contracts.Normalizer[codecNotes], error) {
|
|
||||||
return implementation, nil
|
|
||||||
}); err != nil {
|
|
||||||
t.Fatalf("RegisterNormalizer() error = %v", err)
|
|
||||||
}
|
}
|
||||||
entry, ok := registry.typedEntry(implementation.Key(), artifactKind)
|
|
||||||
if !ok {
|
|
||||||
t.Fatal("typed normalizer entry missing")
|
|
||||||
}
|
|
||||||
built, err := entry.builder(BuildRequest{})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("builder() error = %v", err)
|
|
||||||
}
|
|
||||||
attempts := 3
|
|
||||||
_, err = entry.normalize(context.Background(), built, contracts.TypedNormalizeRequest[any]{
|
|
||||||
MergeOutput: contracts.MergeArtifact[any]{Value: codecNotes{Items: []string{"merged"}}},
|
|
||||||
StructuredOutputRepairAttempts: &attempts,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("normalize() error = %v", err)
|
|
||||||
}
|
|
||||||
if implementation.attempts == nil || *implementation.attempts != attempts {
|
|
||||||
t.Fatalf("repair attempts = %v, want %d", implementation.attempts, attempts)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestTypedRegistryErasurePreservesCorrectionAndCandidateOwnership(t *testing.T) {
|
func registerCollaborationProducer(t *testing.T, stage ModuleStage, impl *collaborationProducer) func(context.Context, contracts.ProducerOperation[any], any) (erasedTypedResult, error) {
|
||||||
const artifactKind contracts.ArtifactKind = "test/notes"
|
t.Helper()
|
||||||
|
spec := ModuleSpec{Key: impl.Key(), Stage: stage, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: "test/value"}
|
||||||
t.Run("extract", func(t *testing.T) {
|
switch stage {
|
||||||
correction := newTestCorrection(t)
|
case StageExtract:
|
||||||
implementation := &correctionObservingNotesExtractor{}
|
|
||||||
registry := NewExtractorRegistry()
|
registry := NewExtractorRegistry()
|
||||||
if err := RegisterExtractor(registry, ModuleSpec{Key: implementation.Key(), Stage: StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: artifactKind}, func() (contracts.Extractor[codecNotes], error) {
|
if err := RegisterExtractor(registry, spec, func() (contracts.Extractor[collaborationValue], error) { return impl, nil }); err != nil {
|
||||||
return implementation, nil
|
t.Fatal(err)
|
||||||
}); err != nil {
|
|
||||||
t.Fatalf("RegisterExtractor() error = %v", err)
|
|
||||||
}
|
}
|
||||||
entry, ok := registry.typedEntry(implementation.Key())
|
entry, _ := registry.typedEntry(spec.Key)
|
||||||
if !ok {
|
return func(ctx context.Context, op contracts.ProducerOperation[any], _ any) (erasedTypedResult, error) {
|
||||||
t.Fatal("typed extractor entry missing")
|
return entry.extract(ctx, impl, contracts.TypedExtractionRequest[any]{Operation: op})
|
||||||
}
|
}
|
||||||
built, err := entry.builder(BuildRequest{})
|
case StageMerge:
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("builder() error = %v", err)
|
|
||||||
}
|
|
||||||
result, err := entry.extract(context.Background(), built, contracts.TypedExtractionRequest{Correction: correction})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("extract() error = %v", err)
|
|
||||||
}
|
|
||||||
assertCorrectionAndCandidateOwnership(t, correction, implementation.correction, implementation.candidate, result.ModelCandidate)
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("merge", func(t *testing.T) {
|
|
||||||
correction := newTestCorrection(t)
|
|
||||||
implementation := &correctionObservingNotesMerger{}
|
|
||||||
registry := NewMergerRegistry()
|
registry := NewMergerRegistry()
|
||||||
if err := RegisterMerger(registry, ModuleSpec{Key: implementation.Key(), Stage: StageMerge, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: artifactKind}, func() (contracts.Merger[codecNotes], error) {
|
if err := RegisterMerger(registry, spec, func() (contracts.Merger[collaborationValue], error) { return impl, nil }); err != nil {
|
||||||
return implementation, nil
|
t.Fatal(err)
|
||||||
}); err != nil {
|
|
||||||
t.Fatalf("RegisterMerger() error = %v", err)
|
|
||||||
}
|
}
|
||||||
entry, ok := registry.typedEntry(implementation.Key(), artifactKind)
|
entry, _ := registry.typedEntry(spec.Key, spec.ArtifactKind)
|
||||||
if !ok {
|
return func(ctx context.Context, op contracts.ProducerOperation[any], value any) (erasedTypedResult, error) {
|
||||||
t.Fatal("typed merger entry missing")
|
return entry.merge(ctx, impl, contracts.TypedMergeRequest[any]{Operation: op, ExtractOutputs: []contracts.ExtractArtifact[any]{{Value: value}}})
|
||||||
}
|
}
|
||||||
built, err := entry.builder(BuildRequest{})
|
default:
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("builder() error = %v", err)
|
|
||||||
}
|
|
||||||
result, err := entry.merge(context.Background(), built, contracts.TypedMergeRequest[any]{Correction: correction, ExtractOutputs: []contracts.ExtractArtifact[any]{{Value: codecNotes{Items: []string{"extract"}}}}})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("merge() error = %v", err)
|
|
||||||
}
|
|
||||||
assertCorrectionAndCandidateOwnership(t, correction, implementation.correction, implementation.candidate, result.ModelCandidate)
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("normalize", func(t *testing.T) {
|
|
||||||
correction := newTestCorrection(t)
|
|
||||||
implementation := &correctionObservingNotesNormalizer{}
|
|
||||||
registry := NewNormalizerRegistry()
|
registry := NewNormalizerRegistry()
|
||||||
if err := RegisterNormalizer(registry, ModuleSpec{Key: implementation.Key(), Stage: StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: artifactKind}, func() (contracts.Normalizer[codecNotes], error) {
|
if err := RegisterNormalizer(registry, spec, func() (contracts.Normalizer[collaborationValue], error) { return impl, nil }); err != nil {
|
||||||
return implementation, nil
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
entry, _ := registry.typedEntry(spec.Key, spec.ArtifactKind)
|
||||||
|
return func(ctx context.Context, op contracts.ProducerOperation[any], value any) (erasedTypedResult, error) {
|
||||||
|
return entry.normalize(ctx, impl, contracts.TypedNormalizeRequest[any]{Operation: op, MergeOutput: contracts.MergeArtifact[any]{Value: value}})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTypedRegistryRejectsIncompatibleOperationValues(t *testing.T) {
|
||||||
|
for _, stage := range []ModuleStage{StageMerge, StageNormalize} {
|
||||||
|
invoke := registerCollaborationProducer(t, stage, &collaborationProducer{})
|
||||||
|
if _, err := invoke(context.Background(), contracts.ProducerOperation[any]{}, struct{ Text string }{"candidate"}); err == nil {
|
||||||
|
t.Fatalf("%s accepted structurally similar but incompatible Go type", stage)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConstructionPassesTheSharedNativeEngine(t *testing.T) {
|
||||||
|
engine := &promptkit.Engine{}
|
||||||
|
registry := NewExtractorRegistry()
|
||||||
|
spec := ModuleSpec{Key: "producer", Stage: StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: "test/value"}
|
||||||
|
if err := RegisterExtractorBuilder(registry, spec, rejectUnconfiguredOptions, func(request BuildRequest) (contracts.Extractor[collaborationValue], error) {
|
||||||
|
if request.Dependencies.Engine != engine {
|
||||||
|
t.Fatal("builder received a different engine")
|
||||||
|
}
|
||||||
|
return &collaborationProducer{}, nil
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
t.Fatalf("RegisterNormalizer() error = %v", err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
entry, ok := registry.typedEntry(implementation.Key(), artifactKind)
|
entry, _ := registry.typedEntry(spec.Key)
|
||||||
if !ok {
|
if _, err := entry.builder(BuildRequest{Dependencies: ModuleDependencies{Engine: engine}}); err != nil {
|
||||||
t.Fatal("typed normalizer entry missing")
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
built, err := entry.builder(BuildRequest{})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("builder() error = %v", err)
|
|
||||||
}
|
|
||||||
result, err := entry.normalize(context.Background(), built, contracts.TypedNormalizeRequest[any]{Correction: correction, MergeOutput: contracts.MergeArtifact[any]{Value: codecNotes{Items: []string{"merge"}}}})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("normalize() error = %v", err)
|
|
||||||
}
|
|
||||||
assertCorrectionAndCandidateOwnership(t, correction, implementation.correction, implementation.candidate, result.ModelCandidate)
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func newTestCorrection(t *testing.T) *contracts.SemanticCorrection {
|
func TestTypedValidatorReceivesNativeExecutionOptions(t *testing.T) {
|
||||||
t.Helper()
|
var got contracts.ExecutionOptions
|
||||||
correction, err := contracts.NewSemanticCorrection([]byte(`{"items":["original"]}`), "Correct the response.")
|
implementation := collaborationValidator{execution: &got}
|
||||||
if err != nil {
|
registry := NewValidatorRegistry()
|
||||||
t.Fatalf("NewSemanticCorrection() error = %v", err)
|
if err := RegisterTypedValidator(registry, "test/value", ValidatorSpec{Key: "assessment", ExecutionClass: contracts.ExecutionClassLLMBacked}, func() (contracts.TypedValidator[collaborationValue], error) { return implementation, nil }); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
return correction
|
entry, _ := registry.typedEntry("assessment", "test/value")
|
||||||
}
|
observed, prepared := false, false
|
||||||
|
native := &promptkit.ExecutionTargetOverride{}
|
||||||
func assertCorrectionAndCandidateOwnership(t *testing.T, callerCorrection, observedCorrection *contracts.SemanticCorrection, producerCandidate, returnedCandidate *contracts.ModelCandidate) {
|
zero := 0
|
||||||
t.Helper()
|
options := contracts.ExecutionOptions{Execution: native, TransientRetries: &zero,
|
||||||
if observedCorrection == nil || producerCandidate == nil || returnedCandidate == nil {
|
Observer: func(promptkit.ExecutionEvent) { observed = true },
|
||||||
t.Fatal("correction and candidates must be present")
|
OnPrepared: func(context.Context, promptkit.PreparedRun) { prepared = true },
|
||||||
}
|
}
|
||||||
callerCorrection.AssistantResponse[0] = '['
|
if _, err := entry.validate(context.Background(), implementation, typedValidationTarget{value: collaborationValue{"candidate"}, execution: options}); err != nil {
|
||||||
if got := string(observedCorrection.AssistantResponse); got != `{"items":["original"]}` {
|
t.Fatal(err)
|
||||||
t.Fatalf("observed correction = %q, want owned original content", got)
|
|
||||||
}
|
}
|
||||||
producerCandidate.Response[0] = '['
|
if got.Execution != native || got.TransientRetries == nil || *got.TransientRetries != 0 || got.Observer == nil || got.OnPrepared == nil {
|
||||||
if bytes.Equal(producerCandidate.Response, returnedCandidate.Response) {
|
t.Fatalf("native options lost: %#v", got)
|
||||||
t.Fatalf("returned candidate aliases producer candidate: %q", returnedCandidate.Response)
|
}
|
||||||
|
got.Observer(promptkit.ExecutionEvent{})
|
||||||
|
got.OnPrepared(context.Background(), promptkit.PreparedRun{})
|
||||||
|
if !observed || !prepared {
|
||||||
|
t.Fatal("native observation callbacks were not retained")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ type typedTestExtractor[T any] struct{ key string }
|
|||||||
|
|
||||||
func (e typedTestExtractor[T]) Key() string { return e.key }
|
func (e typedTestExtractor[T]) Key() string { return e.key }
|
||||||
func (typedTestExtractor[T]) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
func (typedTestExtractor[T]) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||||
func (typedTestExtractor[T]) Extract(context.Context, contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[T], error) {
|
func (typedTestExtractor[T]) Extract(context.Context, contracts.TypedExtractionRequest[T]) (contracts.TypedExtractionResult[T], error) {
|
||||||
return contracts.TypedExtractionResult[T]{}, nil
|
return contracts.TypedExtractionResult[T]{}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -586,3 +586,34 @@ var _ contracts.Normalizer[codecNotes] = typedTestNormalizer[codecNotes]{}
|
|||||||
var _ contracts.TypedValidator[codecNotes] = typedTestValidator[codecNotes]{}
|
var _ contracts.TypedValidator[codecNotes] = typedTestValidator[codecNotes]{}
|
||||||
var _ contracts.ChunkValidator = typedTestChunkValidator{}
|
var _ contracts.ChunkValidator = typedTestChunkValidator{}
|
||||||
var _ contracts.SerializedValidator = typedTestSerializedValidator{}
|
var _ contracts.SerializedValidator = typedTestSerializedValidator{}
|
||||||
|
|
||||||
|
func TestPrepareRetainsInitialAndCorrectionDependencies(t *testing.T) {
|
||||||
|
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
|
||||||
|
const initial = "notarius/test.initial"
|
||||||
|
const correction = "notarius/test.correction"
|
||||||
|
const key = "typed/native-notes"
|
||||||
|
prompts := []string{initial, correction}
|
||||||
|
if err := RegisterExtractor(catalog.Extractors, ModuleSpec{Key: key, Stage: StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: "test/notes", Prompts: prompts}, func() (contracts.Extractor[codecNotes], error) { return typedTestExtractor[codecNotes]{key: key}, nil }); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
profile := typedResolutionProfile()
|
||||||
|
lane := profile.Artifacts["notes"]
|
||||||
|
lane.Extract.Module = key
|
||||||
|
profile.Artifacts["notes"] = lane
|
||||||
|
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
prepared, err := Prepare(resolved, registriesFromModuleCatalog(catalog), ModuleDependencies{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := []string{correction, initial}
|
||||||
|
if got := prepared.Steps[0].ArtifactLanes[0].ExtractPrompts; !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("prepared dependencies = %#v", got)
|
||||||
|
}
|
||||||
|
resolved.Steps[0].ArtifactLanes[0].ExtractPrompts[0] = "changed"
|
||||||
|
if got := prepared.Steps[0].ArtifactLanes[0].Resolved.ExtractPrompts; !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("preparation retained mutable dependencies = %#v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -3,54 +3,34 @@ package pipeline
|
|||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
)
|
)
|
||||||
|
|
||||||
type ExecutionMode string
|
func ParseExecutionMode(value string) (contracts.ExecutionMode, error) {
|
||||||
|
mode := contracts.ExecutionMode(strings.TrimSpace(value))
|
||||||
const (
|
if mode != contracts.ModeStrict && mode != contracts.ModePermissive {
|
||||||
ModeStrict ExecutionMode = "strict"
|
|
||||||
ModePermissive ExecutionMode = "permissive"
|
|
||||||
)
|
|
||||||
|
|
||||||
func ParseExecutionMode(value string) (ExecutionMode, error) {
|
|
||||||
mode := ExecutionMode(strings.TrimSpace(value))
|
|
||||||
if mode != ModeStrict && mode != ModePermissive {
|
|
||||||
return "", fmt.Errorf("mode must be strict or permissive")
|
return "", fmt.Errorf("mode must be strict or permissive")
|
||||||
}
|
}
|
||||||
return mode, nil
|
return mode, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
type CacheMode string
|
func ParseCacheMode(value string) (contracts.CacheMode, error) {
|
||||||
|
mode := contracts.CacheMode(strings.TrimSpace(value))
|
||||||
const (
|
|
||||||
CacheAuto CacheMode = "auto"
|
|
||||||
CacheBypass CacheMode = "bypass"
|
|
||||||
CacheRefresh CacheMode = "refresh"
|
|
||||||
)
|
|
||||||
|
|
||||||
func ParseCacheMode(value string) (CacheMode, error) {
|
|
||||||
mode := CacheMode(strings.TrimSpace(value))
|
|
||||||
switch mode {
|
switch mode {
|
||||||
case CacheAuto, CacheBypass, CacheRefresh:
|
case contracts.CacheAuto, contracts.CacheBypass, contracts.CacheRefresh:
|
||||||
return mode, nil
|
return mode, nil
|
||||||
}
|
}
|
||||||
return "", fmt.Errorf("cache must be auto, bypass, or refresh")
|
return "", fmt.Errorf("cache must be auto, bypass, or refresh")
|
||||||
}
|
}
|
||||||
|
|
||||||
// ProducerControls contains final operation controls; execution does not inherit settings.
|
func ResolveProducerControls(binding ModuleBinding, profile PipelineProfile, invocationCache *contracts.CacheMode) (contracts.ProducerControls, error) {
|
||||||
type ProducerControls struct {
|
result := contracts.ProducerControls{Mode: contracts.ModeStrict, MaxAdditionalTurns: 2, Cache: contracts.CacheAuto}
|
||||||
Mode ExecutionMode `json:"mode"`
|
for _, mode := range []*contracts.ExecutionMode{profile.Mode, binding.Mode} {
|
||||||
MaxAdditionalTurns int `json:"max_additional_turns"`
|
|
||||||
Cache CacheMode `json:"cache"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func ResolveProducerControls(binding ModuleBinding, profile PipelineProfile, invocationCache *CacheMode) (ProducerControls, error) {
|
|
||||||
result := ProducerControls{Mode: ModeStrict, MaxAdditionalTurns: 2, Cache: CacheAuto}
|
|
||||||
for _, mode := range []*ExecutionMode{profile.Mode, binding.Mode} {
|
|
||||||
if mode != nil {
|
if mode != nil {
|
||||||
parsed, err := ParseExecutionMode(string(*mode))
|
parsed, err := ParseExecutionMode(string(*mode))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ProducerControls{}, err
|
return contracts.ProducerControls{}, err
|
||||||
}
|
}
|
||||||
result.Mode = parsed
|
result.Mode = parsed
|
||||||
}
|
}
|
||||||
@@ -58,16 +38,16 @@ func ResolveProducerControls(binding ModuleBinding, profile PipelineProfile, inv
|
|||||||
for _, turns := range []*int{profile.MaxAdditionalTurns, binding.MaxAdditionalTurns} {
|
for _, turns := range []*int{profile.MaxAdditionalTurns, binding.MaxAdditionalTurns} {
|
||||||
if turns != nil {
|
if turns != nil {
|
||||||
if *turns < 0 {
|
if *turns < 0 {
|
||||||
return ProducerControls{}, fmt.Errorf("max_additional_turns must not be negative")
|
return contracts.ProducerControls{}, fmt.Errorf("max_additional_turns must not be negative")
|
||||||
}
|
}
|
||||||
result.MaxAdditionalTurns = *turns
|
result.MaxAdditionalTurns = *turns
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for _, cache := range []*CacheMode{profile.Cache, binding.Cache, invocationCache} {
|
for _, cache := range []*contracts.CacheMode{profile.Cache, binding.Cache, invocationCache} {
|
||||||
if cache != nil {
|
if cache != nil {
|
||||||
parsed, err := ParseCacheMode(string(*cache))
|
parsed, err := ParseCacheMode(string(*cache))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ProducerControls{}, err
|
return contracts.ProducerControls{}, err
|
||||||
}
|
}
|
||||||
result.Cache = parsed
|
result.Cache = parsed
|
||||||
}
|
}
|
||||||
@@ -83,11 +63,11 @@ func cloneControl[T any](value *T) *T {
|
|||||||
return ©
|
return ©
|
||||||
}
|
}
|
||||||
|
|
||||||
func applyEffectiveProducerControls(resolved *ResolvedPipeline, profile PipelineProfile, invocationCache *CacheMode) error {
|
func applyEffectiveProducerControls(resolved *ResolvedPipeline, profile PipelineProfile, invocationCache *contracts.CacheMode) error {
|
||||||
apply := func(binding ModuleBinding, scope string) (ProducerControls, error) {
|
apply := func(binding ModuleBinding, scope string) (contracts.ProducerControls, error) {
|
||||||
controls, err := ResolveProducerControls(binding, profile, invocationCache)
|
controls, err := ResolveProducerControls(binding, profile, invocationCache)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ProducerControls{}, fmt.Errorf("pipeline %q %s: %w", resolved.ID, scope, err)
|
return contracts.ProducerControls{}, fmt.Errorf("pipeline %q %s: %w", resolved.ID, scope, err)
|
||||||
}
|
}
|
||||||
return controls, nil
|
return controls, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,23 +3,25 @@ package pipeline
|
|||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestProducerControlsResolvePrecedenceAndExplicitZero(t *testing.T) {
|
func TestProducerControlsResolvePrecedenceAndExplicitZero(t *testing.T) {
|
||||||
strict, permissive := ModeStrict, ModePermissive
|
strict, permissive := contracts.ModeStrict, contracts.ModePermissive
|
||||||
auto, bypass, refresh := CacheAuto, CacheBypass, CacheRefresh
|
auto, bypass, refresh := contracts.CacheAuto, contracts.CacheBypass, contracts.CacheRefresh
|
||||||
zero, five := 0, 5
|
zero, five := 0, 5
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
profile PipelineProfile
|
profile PipelineProfile
|
||||||
binding ModuleBinding
|
binding ModuleBinding
|
||||||
invocation *CacheMode
|
invocation *contracts.CacheMode
|
||||||
want ProducerControls
|
want contracts.ProducerControls
|
||||||
}{
|
}{
|
||||||
{"defaults", PipelineProfile{}, ModuleBinding{}, nil, ProducerControls{ModeStrict, 2, CacheAuto}},
|
{"defaults", PipelineProfile{}, ModuleBinding{}, nil, contracts.ProducerControls{contracts.ModeStrict, 2, contracts.CacheAuto}},
|
||||||
{"pipeline", PipelineProfile{Mode: &permissive, MaxAdditionalTurns: &five, Cache: &refresh}, ModuleBinding{}, nil, ProducerControls{ModePermissive, 5, CacheRefresh}},
|
{"pipeline", PipelineProfile{Mode: &permissive, MaxAdditionalTurns: &five, Cache: &refresh}, ModuleBinding{}, nil, contracts.ProducerControls{contracts.ModePermissive, 5, contracts.CacheRefresh}},
|
||||||
{"binding", PipelineProfile{Mode: &permissive, MaxAdditionalTurns: &five, Cache: &refresh}, ModuleBinding{Mode: &strict, MaxAdditionalTurns: &zero, Cache: &auto}, nil, ProducerControls{ModeStrict, 0, CacheAuto}},
|
{"binding", PipelineProfile{Mode: &permissive, MaxAdditionalTurns: &five, Cache: &refresh}, ModuleBinding{Mode: &strict, MaxAdditionalTurns: &zero, Cache: &auto}, nil, contracts.ProducerControls{contracts.ModeStrict, 0, contracts.CacheAuto}},
|
||||||
{"invocation", PipelineProfile{Cache: &refresh}, ModuleBinding{Cache: &auto}, &bypass, ProducerControls{ModeStrict, 2, CacheBypass}},
|
{"invocation", PipelineProfile{Cache: &refresh}, ModuleBinding{Cache: &auto}, &bypass, contracts.ProducerControls{contracts.ModeStrict, 2, contracts.CacheBypass}},
|
||||||
}
|
}
|
||||||
for _, test := range tests {
|
for _, test := range tests {
|
||||||
t.Run(test.name, func(t *testing.T) {
|
t.Run(test.name, func(t *testing.T) {
|
||||||
@@ -35,7 +37,7 @@ func TestProducerControlsResolvePrecedenceAndExplicitZero(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestProducerControlsRejectInvalidValuesWithoutMaximumBudgets(t *testing.T) {
|
func TestProducerControlsRejectInvalidValuesWithoutMaximumBudgets(t *testing.T) {
|
||||||
invalidMode, invalidCache := ExecutionMode("unknown"), CacheMode("unknown")
|
invalidMode, invalidCache := contracts.ExecutionMode("unknown"), contracts.CacheMode("unknown")
|
||||||
negative, large := -1, 1000000
|
negative, large := -1, 1000000
|
||||||
for _, binding := range []ModuleBinding{{Mode: &invalidMode}, {Cache: &invalidCache}, {MaxAdditionalTurns: &negative}} {
|
for _, binding := range []ModuleBinding{{Mode: &invalidMode}, {Cache: &invalidCache}, {MaxAdditionalTurns: &negative}} {
|
||||||
if _, err := ResolveProducerControls(binding, PipelineProfile{}, nil); err == nil {
|
if _, err := ResolveProducerControls(binding, PipelineProfile{}, nil); err == nil {
|
||||||
@@ -49,7 +51,7 @@ func TestProducerControlsRejectInvalidValuesWithoutMaximumBudgets(t *testing.T)
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestModuleBindingRoundTripRetainsExplicitControls(t *testing.T) {
|
func TestModuleBindingRoundTripRetainsExplicitControls(t *testing.T) {
|
||||||
strict, bypass, zero := ModeStrict, CacheBypass, 0
|
strict, bypass, zero := contracts.ModeStrict, contracts.CacheBypass, 0
|
||||||
input := ModuleBinding{Module: "fixture", Mode: &strict, Cache: &bypass, MaxAdditionalTurns: &zero}
|
input := ModuleBinding{Module: "fixture", Mode: &strict, Cache: &bypass, MaxAdditionalTurns: &zero}
|
||||||
data, err := json.Marshal(input)
|
data, err := json.Marshal(input)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -66,20 +68,20 @@ func TestModuleBindingRoundTripRetainsExplicitControls(t *testing.T) {
|
|||||||
|
|
||||||
func TestResolvedProducerControlsApplyToEveryLaneOperation(t *testing.T) {
|
func TestResolvedProducerControlsApplyToEveryLaneOperation(t *testing.T) {
|
||||||
profile := baselineProfile()
|
profile := baselineProfile()
|
||||||
permissive, bypass, zero := ModePermissive, CacheBypass, 0
|
permissive, bypass, zero := contracts.ModePermissive, contracts.CacheBypass, 0
|
||||||
profile.Mode = &permissive
|
profile.Mode = &permissive
|
||||||
profile.Chunk.MaxAdditionalTurns = &zero
|
profile.Chunk.MaxAdditionalTurns = &zero
|
||||||
resolved, err := ResolvePipeline(profile, ResolveOptions{CacheOverride: &bypass}, newProfileCatalog(t))
|
resolved, err := ResolvePipeline(profile, ResolveOptions{CacheOverride: &bypass}, newProfileCatalog(t))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if resolved.ChunkControls != (ProducerControls{ModePermissive, 0, CacheBypass}) {
|
if resolved.ChunkControls != (contracts.ProducerControls{contracts.ModePermissive, 0, contracts.CacheBypass}) {
|
||||||
t.Fatalf("chunk controls = %#v", resolved.ChunkControls)
|
t.Fatalf("chunk controls = %#v", resolved.ChunkControls)
|
||||||
}
|
}
|
||||||
for _, step := range resolved.Steps {
|
for _, step := range resolved.Steps {
|
||||||
for _, lane := range step.ArtifactLanes {
|
for _, lane := range step.ArtifactLanes {
|
||||||
for _, got := range []ProducerControls{lane.ExtractControls, lane.MergeControls, lane.NormalizeControls} {
|
for _, got := range []contracts.ProducerControls{lane.ExtractControls, lane.MergeControls, lane.NormalizeControls} {
|
||||||
if got != (ProducerControls{ModePermissive, 2, CacheBypass}) {
|
if got != (contracts.ProducerControls{contracts.ModePermissive, 2, contracts.CacheBypass}) {
|
||||||
t.Fatalf("lane controls = %#v", got)
|
t.Fatalf("lane controls = %#v", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,7 +13,8 @@ import (
|
|||||||
type ValidatorSpec struct {
|
type ValidatorSpec struct {
|
||||||
Key string `json:"key"`
|
Key string `json:"key"`
|
||||||
ExecutionClass contracts.ExecutionClass `json:"execution_class"`
|
ExecutionClass contracts.ExecutionClass `json:"execution_class"`
|
||||||
CorrectionProtocol contracts.CorrectionProtocol `json:"correction_protocol,omitempty"`
|
// Prompts identifies assessment requests, not a producer correction loop.
|
||||||
|
Prompts []string `json:"prompts,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type SerializedValidatorSpec struct {
|
type SerializedValidatorSpec struct {
|
||||||
@@ -116,7 +117,7 @@ func RegisterTypedValidatorBuilder[T any](registry *ValidatorRegistry, kind cont
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return contracts.ValidationResult{}, err
|
return contracts.ValidationResult{}, err
|
||||||
}
|
}
|
||||||
return validator.Validate(ctx, contracts.TypedValidationRequest[T]{Stage: string(target.stage), LaneID: target.laneID, ModuleKey: target.moduleKey, Source: target.source, SourceID: target.sourceID, SourceInput: target.sourceInput, SessionID: target.sessionID, References: target.references, LLMProfile: target.llmProfile, StructuredOutputRepairAttempts: target.structuredOutputRepairAttempts, Metadata: target.metadata, Chunk: target.chunk, Chunks: target.chunks, Ref: target.ref, Value: value})
|
return validator.Validate(ctx, contracts.TypedValidationRequest[T]{Stage: string(target.stage), LaneID: target.laneID, ModuleKey: target.moduleKey, Source: target.source, SourceID: target.sourceID, SourceInput: target.sourceInput, SessionID: target.sessionID, References: target.references, LLMProfile: target.llmProfile, Execution: target.execution, Metadata: target.metadata, Chunk: target.chunk, Chunks: target.chunks, Ref: target.ref, Value: value})
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
@@ -239,14 +240,14 @@ func (r *ValidatorRegistry) Spec(key string) (ValidatorSpec, bool) {
|
|||||||
}
|
}
|
||||||
normalized := strings.TrimSpace(key)
|
normalized := strings.TrimSpace(key)
|
||||||
if entry, found := r.chunkEntries[normalized]; found {
|
if entry, found := r.chunkEntries[normalized]; found {
|
||||||
return entry.spec, true
|
return cloneValidatorSpec(entry.spec), true
|
||||||
}
|
}
|
||||||
if entry, found := r.serializedEntries[normalized]; found {
|
if entry, found := r.serializedEntries[normalized]; found {
|
||||||
return entry.spec.ValidatorSpec, true
|
return cloneValidatorSpec(entry.spec.ValidatorSpec), true
|
||||||
}
|
}
|
||||||
if kinds := r.registeredTypedKinds(normalized); len(kinds) > 0 {
|
if kinds := r.registeredTypedKinds(normalized); len(kinds) > 0 {
|
||||||
entry, found := r.typedEntry(normalized, kinds[0])
|
entry, found := r.typedEntry(normalized, kinds[0])
|
||||||
return entry.spec, found
|
return cloneValidatorSpec(entry.spec), found
|
||||||
}
|
}
|
||||||
return ValidatorSpec{}, false
|
return ValidatorSpec{}, false
|
||||||
}
|
}
|
||||||
@@ -325,7 +326,7 @@ func normalizeValidatorSpec(spec ValidatorSpec) (ValidatorSpec, error) {
|
|||||||
normalized := ValidatorSpec{
|
normalized := ValidatorSpec{
|
||||||
Key: strings.TrimSpace(spec.Key),
|
Key: strings.TrimSpace(spec.Key),
|
||||||
ExecutionClass: spec.ExecutionClass,
|
ExecutionClass: spec.ExecutionClass,
|
||||||
CorrectionProtocol: contracts.CorrectionProtocol(strings.TrimSpace(string(spec.CorrectionProtocol))),
|
Prompts: normalizeStringSet(spec.Prompts),
|
||||||
}
|
}
|
||||||
if normalized.Key == "" {
|
if normalized.Key == "" {
|
||||||
return ValidatorSpec{}, fmt.Errorf("validator key must not be empty")
|
return ValidatorSpec{}, fmt.Errorf("validator key must not be empty")
|
||||||
@@ -335,12 +336,14 @@ func normalizeValidatorSpec(spec ValidatorSpec) (ValidatorSpec, error) {
|
|||||||
default:
|
default:
|
||||||
return ValidatorSpec{}, fmt.Errorf("validator %q execution class %q is not supported", normalized.Key, normalized.ExecutionClass)
|
return ValidatorSpec{}, fmt.Errorf("validator %q execution class %q is not supported", normalized.Key, normalized.ExecutionClass)
|
||||||
}
|
}
|
||||||
if normalized.CorrectionProtocol != "" {
|
|
||||||
return ValidatorSpec{}, fmt.Errorf("validator %q correction protocol is not supported", normalized.Key)
|
|
||||||
}
|
|
||||||
return normalized, nil
|
return normalized, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func sortValidatorSpecs(specs []ValidatorSpec) {
|
func sortValidatorSpecs(specs []ValidatorSpec) {
|
||||||
sort.Slice(specs, func(i, j int) bool { return specs[i].Key < specs[j].Key })
|
sort.Slice(specs, func(i, j int) bool { return specs[i].Key < specs[j].Key })
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func cloneValidatorSpec(spec ValidatorSpec) ValidatorSpec {
|
||||||
|
spec.Prompts = append([]string(nil), spec.Prompts...)
|
||||||
|
return spec
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user