Define typed producer assessment and native execution dependencies
This commit is contained in:
@@ -6,12 +6,13 @@ import (
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
// ModuleDependencies contains run-scoped collaborators shared by constructed
|
||||
// modules. Implementations retain only the dependencies they use.
|
||||
type ModuleDependencies struct {
|
||||
LLM contracts.StructuredLLMClient
|
||||
Engine *promptkit.Engine
|
||||
}
|
||||
|
||||
// BuildRequest contains the stable dependencies and configured options used to
|
||||
|
||||
@@ -53,25 +53,20 @@ func RegisterExtractorBuilder[T any](registry *ExtractorRegistry, spec ModuleSpe
|
||||
if _, ok := registry.specs[normalized.Key]; ok {
|
||||
return fmt.Errorf("extractor %q is already registered", normalized.Key)
|
||||
}
|
||||
entry := typedExtractorEntry{spec: cloneModuleSpec(normalized), valueType: reflect.TypeFor[T](), validateOptions: validateOptions, builder: func(request BuildRequest) (any, error) { return builder(cloneBuildRequest(request)) }, extract: func(ctx context.Context, implementation any, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
|
||||
entry := typedExtractorEntry{spec: cloneModuleSpec(normalized), valueType: reflect.TypeFor[T](), validateOptions: validateOptions, builder: func(request BuildRequest) (any, error) { return builder(cloneBuildRequest(request)) }, extract: func(ctx context.Context, implementation any, request contracts.TypedExtractionRequest[any]) (erasedTypedResult, error) {
|
||||
extractor, ok := implementation.(contracts.Extractor[T])
|
||||
if !ok {
|
||||
return erasedTypedResult{}, fmt.Errorf("extractor %q has incompatible implementation %T", normalized.Key, implementation)
|
||||
}
|
||||
correction, err := contracts.CloneSemanticCorrection(request.Correction)
|
||||
if err != nil {
|
||||
return erasedTypedResult{}, fmt.Errorf("clone extraction correction: %w", err)
|
||||
}
|
||||
request.Correction = correction
|
||||
result, err := extractor.Extract(ctx, request)
|
||||
result, err := extractor.Extract(ctx, contracts.TypedExtractionRequest[T]{
|
||||
Source: request.Source, Chunk: request.Chunk, AmbientContext: request.AmbientContext,
|
||||
SourceInput: request.SourceInput, SessionID: request.SessionID, References: request.References,
|
||||
LLMProfile: request.LLMProfile, Metadata: request.Metadata, Operation: specializeProducerOperation[T](request.Operation),
|
||||
})
|
||||
if err != nil {
|
||||
return erasedTypedResult{}, err
|
||||
}
|
||||
candidate, err := contracts.CloneModelCandidate(result.ModelCandidate)
|
||||
if err != nil {
|
||||
return erasedTypedResult{}, fmt.Errorf("clone extraction model candidate: %w", err)
|
||||
}
|
||||
return erasedTypedResult{Value: result.Value, Diagnostics: contracts.CloneProducerDiagnostics(result.Diagnostics), ModelCandidate: candidate}, nil
|
||||
return erasedTypedResult{Value: result.Value, Diagnostics: contracts.CloneProducerDiagnostics(result.Diagnostics), Outcome: result.Outcome}, nil
|
||||
}}
|
||||
if registry.typedEntries == nil {
|
||||
registry.typedEntries = map[string]typedExtractorEntry{}
|
||||
|
||||
@@ -85,19 +85,14 @@ func RegisterMergerBuilder[T any](registry *MergerRegistry, spec ModuleSpec, val
|
||||
}
|
||||
outputs[i] = contracts.ExtractArtifact[T]{LaneID: output.LaneID, ExtractorKey: output.ExtractorKey, SourceID: output.SourceID, ChunkID: output.ChunkID, ChunkIndex: output.ChunkIndex, ChunkRef: output.ChunkRef, Value: value}
|
||||
}
|
||||
correction, err := contracts.CloneSemanticCorrection(request.Correction)
|
||||
if err != nil {
|
||||
return erasedTypedResult{}, fmt.Errorf("clone merge correction: %w", err)
|
||||
}
|
||||
result, err := merger.Merge(ctx, contracts.TypedMergeRequest[T]{Source: request.Source, LaneID: request.LaneID, ExtractOutputs: outputs, SourceInput: request.SourceInput, SessionID: request.SessionID, References: request.References, LLMProfile: request.LLMProfile, StructuredOutputRepairAttempts: request.StructuredOutputRepairAttempts, Correction: correction, Metadata: request.Metadata})
|
||||
result, err := merger.Merge(ctx, contracts.TypedMergeRequest[T]{
|
||||
Source: request.Source, LaneID: request.LaneID, ExtractOutputs: outputs,
|
||||
SourceInput: request.SourceInput, SessionID: request.SessionID, References: request.References, LLMProfile: request.LLMProfile, Metadata: request.Metadata, Operation: specializeProducerOperation[T](request.Operation),
|
||||
})
|
||||
if err != nil {
|
||||
return erasedTypedResult{}, err
|
||||
}
|
||||
candidate, err := contracts.CloneModelCandidate(result.ModelCandidate)
|
||||
if err != nil {
|
||||
return erasedTypedResult{}, fmt.Errorf("clone merge model candidate: %w", err)
|
||||
}
|
||||
return erasedTypedResult{Value: result.Value, Diagnostics: contracts.CloneProducerDiagnostics(result.Diagnostics), ModelCandidate: candidate}, nil
|
||||
return erasedTypedResult{Value: result.Value, Diagnostics: contracts.CloneProducerDiagnostics(result.Diagnostics), Outcome: result.Outcome}, nil
|
||||
},
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -21,28 +21,29 @@ const (
|
||||
)
|
||||
|
||||
type ModuleSpec struct {
|
||||
Key string
|
||||
Stage ModuleStage
|
||||
ExecutionClass contracts.ExecutionClass
|
||||
CorrectionProtocol contracts.CorrectionProtocol
|
||||
ArtifactKind contracts.ArtifactKind
|
||||
Provides []string
|
||||
Requires []string
|
||||
ReferenceSlots []contracts.ReferenceSlot
|
||||
Key string
|
||||
Stage ModuleStage
|
||||
ExecutionClass contracts.ExecutionClass
|
||||
// Prompts lists statically selected initial/correction IDs shared with the
|
||||
// module's request builders. Dynamic selections are checked at execution.
|
||||
Prompts []string
|
||||
ArtifactKind contracts.ArtifactKind
|
||||
Provides []string
|
||||
Requires []string
|
||||
ReferenceSlots []contracts.ReferenceSlot
|
||||
}
|
||||
|
||||
func normalizeModuleSpec(spec ModuleSpec) ModuleSpec {
|
||||
executionClass := contracts.ExecutionClass(strings.TrimSpace(string(spec.ExecutionClass)))
|
||||
correctionProtocol := contracts.CorrectionProtocol(strings.TrimSpace(string(spec.CorrectionProtocol)))
|
||||
return ModuleSpec{
|
||||
Key: strings.TrimSpace(spec.Key),
|
||||
Stage: spec.Stage,
|
||||
ExecutionClass: executionClass,
|
||||
CorrectionProtocol: correctionProtocol,
|
||||
ArtifactKind: normalizeArtifactKind(spec.ArtifactKind),
|
||||
Provides: normalizeCapabilities(spec.Provides),
|
||||
Requires: normalizeCapabilities(spec.Requires),
|
||||
ReferenceSlots: normalizeReferenceSlots(spec.ReferenceSlots),
|
||||
Key: strings.TrimSpace(spec.Key),
|
||||
Stage: spec.Stage,
|
||||
ExecutionClass: executionClass,
|
||||
Prompts: normalizeStringSet(spec.Prompts),
|
||||
ArtifactKind: normalizeArtifactKind(spec.ArtifactKind),
|
||||
Provides: normalizeCapabilities(spec.Provides),
|
||||
Requires: normalizeCapabilities(spec.Requires),
|
||||
ReferenceSlots: normalizeReferenceSlots(spec.ReferenceSlots),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -73,14 +74,14 @@ func normalizeCapabilities(values []string) []string {
|
||||
|
||||
func cloneModuleSpec(spec ModuleSpec) ModuleSpec {
|
||||
return ModuleSpec{
|
||||
Key: spec.Key,
|
||||
Stage: spec.Stage,
|
||||
ExecutionClass: spec.ExecutionClass,
|
||||
CorrectionProtocol: spec.CorrectionProtocol,
|
||||
ArtifactKind: spec.ArtifactKind,
|
||||
Provides: append([]string(nil), spec.Provides...),
|
||||
Requires: append([]string(nil), spec.Requires...),
|
||||
ReferenceSlots: contracts.CloneReferenceSlots(spec.ReferenceSlots),
|
||||
Key: spec.Key,
|
||||
Stage: spec.Stage,
|
||||
ExecutionClass: spec.ExecutionClass,
|
||||
Prompts: append([]string(nil), spec.Prompts...),
|
||||
ArtifactKind: spec.ArtifactKind,
|
||||
Provides: append([]string(nil), spec.Provides...),
|
||||
Requires: append([]string(nil), spec.Requires...),
|
||||
ReferenceSlots: contracts.CloneReferenceSlots(spec.ReferenceSlots),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -97,19 +98,6 @@ func validateModuleSpec(kind string, expectedStage ModuleStage, spec ModuleSpec)
|
||||
if spec.ExecutionClass != contracts.ExecutionClassDeterministic && spec.ExecutionClass != contracts.ExecutionClassLLMBacked {
|
||||
return fmt.Errorf("%s %q has unsupported execution class %q", kind, spec.Key, spec.ExecutionClass)
|
||||
}
|
||||
if spec.CorrectionProtocol != "" {
|
||||
if err := spec.CorrectionProtocol.Validate(); err != nil {
|
||||
return fmt.Errorf("%s %q correction protocol: %w", kind, spec.Key, err)
|
||||
}
|
||||
if spec.ExecutionClass != contracts.ExecutionClassLLMBacked {
|
||||
return fmt.Errorf("%s %q correction protocol requires an LLM-backed execution class", kind, spec.Key)
|
||||
}
|
||||
switch spec.Stage {
|
||||
case StageChunk, StageExtract, StageMerge, StageNormalize:
|
||||
default:
|
||||
return fmt.Errorf("%s %q correction protocol is not supported for %q stage", kind, spec.Key, spec.Stage)
|
||||
}
|
||||
}
|
||||
if spec.ArtifactKind != "" && spec.Stage != StageExtract && spec.Stage != StageMerge && spec.Stage != StageNormalize {
|
||||
return fmt.Errorf("%s %q must not declare an artifact kind", kind, spec.Key)
|
||||
}
|
||||
|
||||
@@ -32,67 +32,41 @@ func TestValidateModuleSpecRequiresSupportedExecutionClass(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloneModuleSpecPreservesCorrectionProtocol(t *testing.T) {
|
||||
spec := normalizeModuleSpec(ModuleSpec{
|
||||
Key: " module ",
|
||||
Stage: StageChunk,
|
||||
ExecutionClass: contracts.ExecutionClassLLMBacked,
|
||||
CorrectionProtocol: " single_response_v1 ",
|
||||
})
|
||||
if spec.CorrectionProtocol != contracts.CorrectionProtocolSingleResponseV1 {
|
||||
t.Fatalf("normalized CorrectionProtocol = %q, want %q", spec.CorrectionProtocol, contracts.CorrectionProtocolSingleResponseV1)
|
||||
func TestModuleSpecRetainsDetachedStaticPromptDependencies(t *testing.T) {
|
||||
const initial = "notarius/test.initial"
|
||||
const correction = "notarius/test.correction"
|
||||
prompts := []string{correction, initial}
|
||||
registry := NewExtractorRegistry()
|
||||
if err := RegisterExtractor(registry, ModuleSpec{Key: "producer", Stage: StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: "test/value", Prompts: prompts}, func() (contracts.Extractor[collaborationValue], error) { return &collaborationProducer{}, nil }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cloned := cloneModuleSpec(spec)
|
||||
if !reflect.DeepEqual(cloned, spec) {
|
||||
t.Fatalf("cloneModuleSpec() = %#v, want %#v", cloned, spec)
|
||||
prompts[0] = "changed"
|
||||
spec, ok := registry.Spec("producer")
|
||||
if !ok || !reflect.DeepEqual(spec.Prompts, []string{correction, initial}) {
|
||||
t.Fatalf("static dependencies = %#v", spec.Prompts)
|
||||
}
|
||||
spec.Prompts[0] = "also changed"
|
||||
again, _ := registry.Spec("producer")
|
||||
if !reflect.DeepEqual(again.Prompts, []string{correction, initial}) {
|
||||
t.Fatalf("registry declarations mutated: %#v", again.Prompts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateModuleSpecCorrectionProtocolEligibility(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
stage ModuleStage
|
||||
class contracts.ExecutionClass
|
||||
protocol contracts.CorrectionProtocol
|
||||
want string
|
||||
}{
|
||||
{name: "LLM chunk", stage: StageChunk, class: contracts.ExecutionClassLLMBacked, protocol: contracts.CorrectionProtocolSingleResponseV1},
|
||||
{name: "LLM extract", stage: StageExtract, class: contracts.ExecutionClassLLMBacked, protocol: contracts.CorrectionProtocolSingleResponseV1},
|
||||
{name: "LLM merge", stage: StageMerge, class: contracts.ExecutionClassLLMBacked, protocol: contracts.CorrectionProtocolSingleResponseV1},
|
||||
{name: "LLM normalize", stage: StageNormalize, class: contracts.ExecutionClassLLMBacked, protocol: contracts.CorrectionProtocolSingleResponseV1},
|
||||
{name: "deterministic chunk", stage: StageChunk, class: contracts.ExecutionClassDeterministic, protocol: contracts.CorrectionProtocolSingleResponseV1, want: "LLM-backed"},
|
||||
{name: "input", stage: StageInput, class: contracts.ExecutionClassLLMBacked, protocol: contracts.CorrectionProtocolSingleResponseV1, want: "not supported"},
|
||||
{name: "output", stage: StageOutput, class: contracts.ExecutionClassLLMBacked, protocol: contracts.CorrectionProtocolSingleResponseV1, want: "not supported"},
|
||||
{name: "unknown protocol", stage: StageChunk, class: contracts.ExecutionClassLLMBacked, protocol: "unsupported", want: "unsupported correction protocol"},
|
||||
func TestValidatorSpecRetainsDetachedPromptDependencies(t *testing.T) {
|
||||
prompts := []string{"notarius/assessment"}
|
||||
registry := NewValidatorRegistry()
|
||||
if err := RegisterTypedValidator(registry, "test/value", ValidatorSpec{Key: "assessment", ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: prompts}, func() (contracts.TypedValidator[collaborationValue], error) { return collaborationValidator{}, nil }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
spec := normalizeModuleSpec(ModuleSpec{
|
||||
Key: "module",
|
||||
Stage: test.stage,
|
||||
ExecutionClass: test.class,
|
||||
CorrectionProtocol: test.protocol,
|
||||
})
|
||||
err := validateModuleSpec("module", test.stage, spec)
|
||||
if test.want == "" && err != nil {
|
||||
t.Fatalf("validateModuleSpec() error = %v, want nil", err)
|
||||
}
|
||||
if test.want != "" && (err == nil || !strings.Contains(err.Error(), test.want)) {
|
||||
t.Fatalf("validateModuleSpec() error = %v, want %q", err, test.want)
|
||||
}
|
||||
})
|
||||
prompts[0] = "changed"
|
||||
spec, ok := registry.Spec("assessment")
|
||||
if !ok || !reflect.DeepEqual(spec.Prompts, []string{"notarius/assessment"}) {
|
||||
t.Fatalf("validator dependencies = %#v", spec.Prompts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeValidatorSpecRejectsCorrectionProtocol(t *testing.T) {
|
||||
_, err := normalizeValidatorSpec(ValidatorSpec{
|
||||
Key: "validator",
|
||||
ExecutionClass: contracts.ExecutionClassLLMBacked,
|
||||
CorrectionProtocol: contracts.CorrectionProtocolSingleResponseV1,
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "correction protocol") {
|
||||
t.Fatalf("normalizeValidatorSpec() error = %v, want correction protocol error", err)
|
||||
spec.Prompts[0] = "also changed"
|
||||
again, _ := registry.Spec("assessment")
|
||||
if !reflect.DeepEqual(again.Prompts, []string{"notarius/assessment"}) {
|
||||
t.Fatalf("validator declarations mutated: %#v", again.Prompts)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -76,36 +76,19 @@ func RegisterNormalizerBuilder[T any](registry *NormalizerRegistry, spec ModuleS
|
||||
if err != nil {
|
||||
return erasedTypedResult{}, err
|
||||
}
|
||||
correction, err := contracts.CloneSemanticCorrection(request.Correction)
|
||||
if err != nil {
|
||||
return erasedTypedResult{}, fmt.Errorf("clone normalize correction: %w", err)
|
||||
}
|
||||
result, err := normalizer.Normalize(ctx, contracts.TypedNormalizeRequest[T]{Source: request.Source, LaneID: request.LaneID, MergeOutput: contracts.MergeArtifact[T]{LaneID: request.MergeOutput.LaneID, MergerKey: request.MergeOutput.MergerKey, SourceID: request.MergeOutput.SourceID, Value: value}, SourceInput: request.SourceInput, SessionID: request.SessionID, References: request.References, LLMProfile: request.LLMProfile, StructuredOutputRepairAttempts: request.StructuredOutputRepairAttempts, Correction: correction, Metadata: request.Metadata})
|
||||
result, err := normalizer.Normalize(ctx, contracts.TypedNormalizeRequest[T]{
|
||||
Source: request.Source, LaneID: request.LaneID, MergeOutput: contracts.MergeArtifact[T]{LaneID: request.MergeOutput.LaneID, MergerKey: request.MergeOutput.MergerKey, SourceID: request.MergeOutput.SourceID, Value: value},
|
||||
SourceInput: request.SourceInput, SessionID: request.SessionID, References: request.References, LLMProfile: request.LLMProfile, Metadata: request.Metadata, Operation: specializeProducerOperation[T](request.Operation),
|
||||
})
|
||||
if err != nil {
|
||||
return erasedTypedResult{}, err
|
||||
}
|
||||
candidate, err := contracts.CloneModelCandidate(result.ModelCandidate)
|
||||
if err != nil {
|
||||
return erasedTypedResult{}, fmt.Errorf("clone normalize model candidate: %w", err)
|
||||
}
|
||||
return erasedTypedResult{Value: result.Value, Diagnostics: contracts.CloneProducerDiagnostics(result.Diagnostics), Retry: cloneNormalizeRetry(result.Retry), ModelCandidate: candidate}, nil
|
||||
return erasedTypedResult{Value: result.Value, Diagnostics: contracts.CloneProducerDiagnostics(result.Diagnostics), Outcome: result.Outcome}, nil
|
||||
},
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func cloneNormalizeRetry(retry *contracts.NormalizeRetry) *contracts.NormalizeRetry {
|
||||
if retry == nil {
|
||||
return nil
|
||||
}
|
||||
return &contracts.NormalizeRetry{
|
||||
ReasonCode: retry.ReasonCode,
|
||||
Message: retry.Message,
|
||||
CorrectionGuidance: retry.CorrectionGuidance,
|
||||
FallbackDiagnostics: contracts.CloneProducerDiagnostics(retry.FallbackDiagnostics),
|
||||
}
|
||||
}
|
||||
|
||||
func (r *NormalizerRegistry) validateOptions(key string, kind contracts.ArtifactKind, options map[string]any) error {
|
||||
if r == nil {
|
||||
return fmt.Errorf("normalizer registry must not be nil")
|
||||
|
||||
@@ -13,11 +13,11 @@ import (
|
||||
// resolved pipeline. Its implementation values are private so execution cannot
|
||||
// replace or reconfigure them after preparation.
|
||||
type PreparedPipeline struct {
|
||||
Input ModuleBinding
|
||||
Chunk ModuleBinding
|
||||
ChunkCorrectionProtocol contracts.CorrectionProtocol
|
||||
Steps []PreparedPipelineStep
|
||||
Output ModuleBinding
|
||||
Input ModuleBinding
|
||||
Chunk ModuleBinding
|
||||
ChunkPrompts []string
|
||||
Steps []PreparedPipelineStep
|
||||
Output ModuleBinding
|
||||
|
||||
resolved ResolvedPipeline
|
||||
dependencies ModuleDependencies
|
||||
@@ -37,10 +37,10 @@ type PreparedPipelineStep struct {
|
||||
}
|
||||
|
||||
type PreparedArtifactLane struct {
|
||||
Resolved ResolvedArtifactLane
|
||||
ExtractCorrectionProtocol contracts.CorrectionProtocol
|
||||
MergeCorrectionProtocol contracts.CorrectionProtocol
|
||||
NormalizeCorrectionProtocol contracts.CorrectionProtocol
|
||||
Resolved ResolvedArtifactLane
|
||||
ExtractPrompts []string
|
||||
MergePrompts []string
|
||||
NormalizePrompts []string
|
||||
}
|
||||
|
||||
type preparedLaneExecutor struct {
|
||||
@@ -92,21 +92,18 @@ func Prepare(resolved ResolvedPipeline, registries Registries, deps ModuleDepend
|
||||
if err := validateResolvedPipeline(resolved); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := validateCorrectionRetryCapabilities(resolved); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := validateRegistrySet(resolved, registries); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stable := cloneResolvedPipeline(resolved)
|
||||
prepared := &PreparedPipeline{
|
||||
Input: cloneModuleBinding(stable.Input),
|
||||
Chunk: cloneModuleBinding(stable.Chunk),
|
||||
ChunkCorrectionProtocol: stable.ChunkCorrectionProtocol,
|
||||
Output: cloneModuleBinding(stable.Output),
|
||||
resolved: stable,
|
||||
dependencies: deps,
|
||||
artifactCodecs: registries.ArtifactCodecs,
|
||||
Input: cloneModuleBinding(stable.Input),
|
||||
Chunk: cloneModuleBinding(stable.Chunk),
|
||||
ChunkPrompts: append([]string(nil), stable.ChunkPrompts...),
|
||||
Output: cloneModuleBinding(stable.Output),
|
||||
resolved: stable,
|
||||
dependencies: deps,
|
||||
artifactCodecs: registries.ArtifactCodecs,
|
||||
}
|
||||
request := func(binding ModuleBinding, references contracts.ReferenceSet) BuildRequest {
|
||||
return BuildRequest{Dependencies: deps, Options: binding.Options, References: references}
|
||||
@@ -141,10 +138,10 @@ func Prepare(resolved ResolvedPipeline, registries Registries, deps ModuleDepend
|
||||
return nil, err
|
||||
}
|
||||
preparedStep.ArtifactLanes = append(preparedStep.ArtifactLanes, PreparedArtifactLane{
|
||||
Resolved: cloneResolvedArtifactLane(lane),
|
||||
ExtractCorrectionProtocol: lane.ExtractCorrectionProtocol,
|
||||
MergeCorrectionProtocol: lane.MergeCorrectionProtocol,
|
||||
NormalizeCorrectionProtocol: lane.NormalizeCorrectionProtocol,
|
||||
Resolved: cloneResolvedArtifactLane(lane),
|
||||
ExtractPrompts: append([]string(nil), lane.ExtractPrompts...),
|
||||
MergePrompts: append([]string(nil), lane.MergePrompts...),
|
||||
NormalizePrompts: append([]string(nil), lane.NormalizePrompts...),
|
||||
})
|
||||
preparedStep.lanes = append(preparedStep.lanes, executor)
|
||||
}
|
||||
@@ -213,40 +210,6 @@ func prepareEvidencePlan(resolved ResolvedPipeline, registries Registries, outpu
|
||||
return plan, nil
|
||||
}
|
||||
|
||||
func validateCorrectionRetryCapabilities(pipeline ResolvedPipeline) error {
|
||||
validate := func(stage ModuleStage, laneID string, binding ModuleBinding, executionClass contracts.ExecutionClass, protocol contracts.CorrectionProtocol) error {
|
||||
if executionClass != contracts.ExecutionClassLLMBacked || binding.Retries == 0 {
|
||||
return nil
|
||||
}
|
||||
chain := resolvedValidatorChain(stage, laneID, binding.Module, pipeline.ValidatorChains)
|
||||
if len(chain.Validators) == 0 || protocol == contracts.CorrectionProtocolSingleResponseV1 {
|
||||
return nil
|
||||
}
|
||||
if laneID == "" {
|
||||
return fmt.Errorf("pipeline %q %s module %q configures validators and retries but does not declare correction protocol %q", pipeline.ID, stage, binding.Module, contracts.CorrectionProtocolSingleResponseV1)
|
||||
}
|
||||
return fmt.Errorf("pipeline %q lane %q %s module %q configures validators and retries but does not declare correction protocol %q", pipeline.ID, laneID, stage, binding.Module, contracts.CorrectionProtocolSingleResponseV1)
|
||||
}
|
||||
|
||||
if err := validate(StageChunk, "", pipeline.Chunk, pipeline.ChunkExecutionClass, pipeline.ChunkCorrectionProtocol); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, step := range pipeline.Steps {
|
||||
for _, lane := range step.ArtifactLanes {
|
||||
if err := validate(StageExtract, lane.ID, lane.Extract, lane.ExtractExecutionClass, lane.ExtractCorrectionProtocol); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validate(StageMerge, lane.ID, lane.Merge, lane.MergeExecutionClass, lane.MergeCorrectionProtocol); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validate(StageNormalize, lane.ID, lane.Normalize, lane.NormalizeExecutionClass, lane.NormalizeCorrectionProtocol); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func prepareLane(pipeline ResolvedPipeline, lane ResolvedArtifactLane, registries Registries, deps ModuleDependencies) (preparedLaneExecutor, error) {
|
||||
executor := preparedLaneExecutor{resolved: cloneResolvedArtifactLane(lane)}
|
||||
request := func(binding ModuleBinding, references contracts.ReferenceSet) BuildRequest {
|
||||
@@ -454,7 +417,7 @@ func validateRegistrySet(resolved ResolvedPipeline, registries Registries) error
|
||||
|
||||
func cloneResolvedPipeline(in ResolvedPipeline) ResolvedPipeline {
|
||||
out := in
|
||||
out.ConfiguredValidationPolicy = cloneValidationPolicyOverride(in.ConfiguredValidationPolicy)
|
||||
out.ChunkPrompts = append([]string(nil), in.ChunkPrompts...)
|
||||
out.Input = cloneModuleBinding(in.Input)
|
||||
out.Chunk = cloneModuleBinding(in.Chunk)
|
||||
out.Output = cloneModuleBinding(in.Output)
|
||||
@@ -477,6 +440,9 @@ func cloneResolvedPipeline(in ResolvedPipeline) ResolvedPipeline {
|
||||
|
||||
func cloneResolvedArtifactLane(in ResolvedArtifactLane) ResolvedArtifactLane {
|
||||
out := in
|
||||
out.ExtractPrompts = append([]string(nil), in.ExtractPrompts...)
|
||||
out.MergePrompts = append([]string(nil), in.MergePrompts...)
|
||||
out.NormalizePrompts = append([]string(nil), in.NormalizePrompts...)
|
||||
out.Extract = cloneModuleBinding(in.Extract)
|
||||
out.Merge = cloneModuleBinding(in.Merge)
|
||||
out.Normalize = cloneModuleBinding(in.Normalize)
|
||||
|
||||
@@ -21,9 +21,9 @@ const (
|
||||
)
|
||||
|
||||
type ModuleBinding struct {
|
||||
Mode *ExecutionMode `json:"mode,omitempty"`
|
||||
Mode *contracts.ExecutionMode `json:"mode,omitempty"`
|
||||
MaxAdditionalTurns *int `json:"max_additional_turns,omitempty"`
|
||||
Cache *CacheMode `json:"cache,omitempty"`
|
||||
Cache *contracts.CacheMode `json:"cache,omitempty"`
|
||||
Module string `json:"module"`
|
||||
LLMProfile string `json:"llm_profile,omitempty"`
|
||||
Options map[string]any `json:"options,omitempty"`
|
||||
@@ -82,9 +82,9 @@ type ValidatorOverride struct {
|
||||
|
||||
func (binding ModuleBinding) MarshalJSON() ([]byte, error) {
|
||||
type moduleBindingJSON struct {
|
||||
Mode *ExecutionMode `json:"mode,omitempty"`
|
||||
Mode *contracts.ExecutionMode `json:"mode,omitempty"`
|
||||
MaxAdditionalTurns *int `json:"max_additional_turns,omitempty"`
|
||||
Cache *CacheMode `json:"cache,omitempty"`
|
||||
Cache *contracts.CacheMode `json:"cache,omitempty"`
|
||||
Module string `json:"module"`
|
||||
LLMProfile string `json:"llm_profile,omitempty"`
|
||||
Options map[string]any `json:"options,omitempty"`
|
||||
@@ -112,9 +112,9 @@ func (binding *ModuleBinding) UnmarshalJSON(data []byte) error {
|
||||
return fmt.Errorf("module binding must not be nil")
|
||||
}
|
||||
type moduleBindingJSON struct {
|
||||
Mode *ExecutionMode `json:"mode,omitempty"`
|
||||
Mode *contracts.ExecutionMode `json:"mode,omitempty"`
|
||||
MaxAdditionalTurns *int `json:"max_additional_turns,omitempty"`
|
||||
Cache *CacheMode `json:"cache,omitempty"`
|
||||
Cache *contracts.CacheMode `json:"cache,omitempty"`
|
||||
Module string `json:"module"`
|
||||
LLMProfile string `json:"llm_profile,omitempty"`
|
||||
Options map[string]any `json:"options,omitempty"`
|
||||
@@ -160,9 +160,9 @@ type PipelineStepProfile struct {
|
||||
}
|
||||
|
||||
type PipelineProfile struct {
|
||||
Mode *ExecutionMode `json:"mode,omitempty"`
|
||||
Mode *contracts.ExecutionMode `json:"mode,omitempty"`
|
||||
MaxAdditionalTurns *int `json:"max_additional_turns,omitempty"`
|
||||
Cache *CacheMode `json:"cache,omitempty"`
|
||||
Cache *contracts.CacheMode `json:"cache,omitempty"`
|
||||
ID string `json:"id"`
|
||||
LLMProfile string `json:"llm_profile,omitempty"`
|
||||
Input ModuleBinding `json:"input"`
|
||||
@@ -174,7 +174,7 @@ type PipelineProfile struct {
|
||||
}
|
||||
|
||||
type ResolveOptions struct {
|
||||
CacheOverride *CacheMode
|
||||
CacheOverride *contracts.CacheMode
|
||||
Only []string
|
||||
LLMProfileOverride string
|
||||
ReferenceOverrides []ReferenceBinding
|
||||
@@ -206,29 +206,29 @@ type ResolvedReferenceTarget struct {
|
||||
}
|
||||
|
||||
type ResolvedArtifactLane struct {
|
||||
ExtractControls ProducerControls
|
||||
MergeControls ProducerControls
|
||||
NormalizeControls ProducerControls
|
||||
StepID string
|
||||
ID string
|
||||
ArtifactKind contracts.ArtifactKind `json:"artifact_kind,omitempty"`
|
||||
ArtifactSchemaID string `json:"artifact_schema_id,omitempty"`
|
||||
ArtifactSchemaName string `json:"artifact_schema_name,omitempty"`
|
||||
ArtifactSchemaVersion string `json:"artifact_schema_version,omitempty"`
|
||||
ArtifactSchemaDigest string `json:"artifact_schema_digest,omitempty"`
|
||||
Extract ModuleBinding
|
||||
ExtractExecutionClass contracts.ExecutionClass `json:"extract_execution_class"`
|
||||
ExtractCorrectionProtocol contracts.CorrectionProtocol `json:"extract_correction_protocol,omitempty"`
|
||||
Merge ModuleBinding
|
||||
MergeExecutionClass contracts.ExecutionClass `json:"merge_execution_class"`
|
||||
MergeCorrectionProtocol contracts.CorrectionProtocol `json:"merge_correction_protocol,omitempty"`
|
||||
Normalize ModuleBinding
|
||||
NormalizeExecutionClass contracts.ExecutionClass `json:"normalize_execution_class"`
|
||||
NormalizeCorrectionProtocol contracts.CorrectionProtocol `json:"normalize_correction_protocol,omitempty"`
|
||||
Validators []ModuleBinding
|
||||
ExtractReferences ResolvedReferenceTarget `json:"extract_references"`
|
||||
MergeReferences ResolvedReferenceTarget `json:"merge_references"`
|
||||
NormalizeReferences ResolvedReferenceTarget `json:"normalize_references"`
|
||||
ExtractControls contracts.ProducerControls
|
||||
MergeControls contracts.ProducerControls
|
||||
NormalizeControls contracts.ProducerControls
|
||||
StepID string
|
||||
ID string
|
||||
ArtifactKind contracts.ArtifactKind `json:"artifact_kind,omitempty"`
|
||||
ArtifactSchemaID string `json:"artifact_schema_id,omitempty"`
|
||||
ArtifactSchemaName string `json:"artifact_schema_name,omitempty"`
|
||||
ArtifactSchemaVersion string `json:"artifact_schema_version,omitempty"`
|
||||
ArtifactSchemaDigest string `json:"artifact_schema_digest,omitempty"`
|
||||
Extract ModuleBinding
|
||||
ExtractExecutionClass contracts.ExecutionClass `json:"extract_execution_class"`
|
||||
ExtractPrompts []string
|
||||
Merge ModuleBinding
|
||||
MergeExecutionClass contracts.ExecutionClass `json:"merge_execution_class"`
|
||||
MergePrompts []string
|
||||
Normalize ModuleBinding
|
||||
NormalizeExecutionClass contracts.ExecutionClass `json:"normalize_execution_class"`
|
||||
NormalizePrompts []string
|
||||
Validators []ModuleBinding
|
||||
ExtractReferences ResolvedReferenceTarget `json:"extract_references"`
|
||||
MergeReferences ResolvedReferenceTarget `json:"merge_references"`
|
||||
NormalizeReferences ResolvedReferenceTarget `json:"normalize_references"`
|
||||
}
|
||||
|
||||
type ResolvedPipelineStep struct {
|
||||
@@ -244,6 +244,7 @@ type ResolvedValidatorChain struct {
|
||||
}
|
||||
|
||||
type ResolvedValidator struct {
|
||||
Prompts []string `json:"prompts,omitempty"`
|
||||
Binding ModuleBinding `json:"binding"`
|
||||
ExecutionClass contracts.ExecutionClass `json:"execution_class"`
|
||||
Target ValidatorTarget `json:"target,omitempty"`
|
||||
@@ -251,19 +252,19 @@ type ResolvedValidator struct {
|
||||
}
|
||||
|
||||
type ResolvedPipeline struct {
|
||||
ChunkControls ProducerControls
|
||||
ID string
|
||||
Digest string
|
||||
Input ModuleBinding
|
||||
InputExecutionClass contracts.ExecutionClass `json:"input_execution_class"`
|
||||
Chunk ModuleBinding
|
||||
ChunkExecutionClass contracts.ExecutionClass `json:"chunk_execution_class"`
|
||||
ChunkCorrectionProtocol contracts.CorrectionProtocol `json:"chunk_correction_protocol,omitempty"`
|
||||
ChunkReferences ResolvedReferenceTarget `json:"chunk_references"`
|
||||
Steps []ResolvedPipelineStep
|
||||
ValidatorChains []ResolvedValidatorChain `json:"validator_chains"`
|
||||
Output ModuleBinding
|
||||
OutputExecutionClass contracts.ExecutionClass `json:"output_execution_class"`
|
||||
ChunkControls contracts.ProducerControls
|
||||
ID string
|
||||
Digest string
|
||||
Input ModuleBinding
|
||||
InputExecutionClass contracts.ExecutionClass `json:"input_execution_class"`
|
||||
Chunk ModuleBinding
|
||||
ChunkExecutionClass contracts.ExecutionClass `json:"chunk_execution_class"`
|
||||
ChunkPrompts []string
|
||||
ChunkReferences ResolvedReferenceTarget `json:"chunk_references"`
|
||||
Steps []ResolvedPipelineStep
|
||||
ValidatorChains []ResolvedValidatorChain `json:"validator_chains"`
|
||||
Output ModuleBinding
|
||||
OutputExecutionClass contracts.ExecutionClass `json:"output_execution_class"`
|
||||
}
|
||||
|
||||
// AllArtifactLanes returns lanes in deterministic step order for read-only
|
||||
@@ -430,14 +431,14 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
|
||||
return ResolvedPipeline{}, err
|
||||
}
|
||||
resolved := ResolvedPipeline{
|
||||
ID: pipelineID,
|
||||
Input: input,
|
||||
InputExecutionClass: inputModuleSpec.ExecutionClass,
|
||||
Chunk: chunk,
|
||||
ChunkExecutionClass: chunkSpec.ExecutionClass,
|
||||
ChunkCorrectionProtocol: chunkSpec.CorrectionProtocol,
|
||||
ChunkReferences: referenceTarget(StageChunk, "", chunk.Module, chunkReferences),
|
||||
Output: output,
|
||||
ID: pipelineID,
|
||||
Input: input,
|
||||
InputExecutionClass: inputModuleSpec.ExecutionClass,
|
||||
Chunk: chunk,
|
||||
ChunkExecutionClass: chunkSpec.ExecutionClass,
|
||||
ChunkPrompts: append([]string(nil), chunkSpec.Prompts...),
|
||||
ChunkReferences: referenceTarget(StageChunk, "", chunk.Module, chunkReferences),
|
||||
Output: output,
|
||||
}
|
||||
chunkValidatorChain, err := resolveValidatorChain(pipelineID, "", StageChunk, chunk.Module, chunk.Validators, "", nil, catalog)
|
||||
if err != nil {
|
||||
@@ -617,7 +618,7 @@ func resolveArtifactLane(
|
||||
lane.ExtractReferences = referenceTarget(StageExtract, laneID, lane.Extract.Module, references)
|
||||
lane.ExtractReferences.StepID = strings.TrimSpace(stepID)
|
||||
lane.ExtractExecutionClass = extractSpec.ExecutionClass
|
||||
lane.ExtractCorrectionProtocol = extractSpec.CorrectionProtocol
|
||||
lane.ExtractPrompts = append([]string(nil), extractSpec.Prompts...)
|
||||
capabilities.add(extractSpec.Provides...)
|
||||
|
||||
mergeSpec, err := mergerSpecForArtifact(catalog, lane.Merge.Module, lane.ArtifactKind, artifactType)
|
||||
@@ -644,7 +645,7 @@ func resolveArtifactLane(
|
||||
lane.MergeReferences = referenceTarget(StageMerge, laneID, lane.Merge.Module, mergeReferences)
|
||||
lane.MergeReferences.StepID = strings.TrimSpace(stepID)
|
||||
lane.MergeExecutionClass = mergeSpec.ExecutionClass
|
||||
lane.MergeCorrectionProtocol = mergeSpec.CorrectionProtocol
|
||||
lane.MergePrompts = append([]string(nil), mergeSpec.Prompts...)
|
||||
capabilities.add(mergeSpec.Provides...)
|
||||
|
||||
normalizeSpec, err := normalizerSpecForArtifact(catalog, lane.Normalize.Module, lane.ArtifactKind, artifactType)
|
||||
@@ -671,7 +672,7 @@ func resolveArtifactLane(
|
||||
lane.NormalizeReferences = referenceTarget(StageNormalize, laneID, lane.Normalize.Module, normalizeReferences)
|
||||
lane.NormalizeReferences.StepID = strings.TrimSpace(stepID)
|
||||
lane.NormalizeExecutionClass = normalizeSpec.ExecutionClass
|
||||
lane.NormalizeCorrectionProtocol = normalizeSpec.CorrectionProtocol
|
||||
lane.NormalizePrompts = append([]string(nil), normalizeSpec.Prompts...)
|
||||
capabilities.add(normalizeSpec.Provides...)
|
||||
|
||||
if len(lane.Validators) > 0 {
|
||||
@@ -936,6 +937,7 @@ func resolveValidatorChain(pipelineID string, laneID string, stage ModuleStage,
|
||||
chain.Validators = append(chain.Validators, ResolvedValidator{
|
||||
Binding: cloneModuleBinding(validator),
|
||||
ExecutionClass: spec.ExecutionClass,
|
||||
Prompts: append([]string(nil), spec.Prompts...),
|
||||
Target: target,
|
||||
ArtifactKind: artifactKind,
|
||||
})
|
||||
@@ -968,6 +970,7 @@ func cloneResolvedValidators(validators []ResolvedValidator) []ResolvedValidator
|
||||
out[i] = ResolvedValidator{
|
||||
Binding: cloneModuleBinding(validator.Binding),
|
||||
ExecutionClass: validator.ExecutionClass,
|
||||
Prompts: append([]string(nil), validator.Prompts...),
|
||||
Target: validator.Target,
|
||||
ArtifactKind: validator.ArtifactKind,
|
||||
}
|
||||
@@ -1517,31 +1520,31 @@ func selectedArtifactLanes(pipelineID string, artifacts map[string]ArtifactLaneP
|
||||
|
||||
func resolvedPipelineDigest(resolved ResolvedPipeline) (string, error) {
|
||||
withoutDigest := struct {
|
||||
ChunkControls ProducerControls
|
||||
ID string
|
||||
Input ModuleBinding
|
||||
InputExecutionClass contracts.ExecutionClass
|
||||
Chunk ModuleBinding
|
||||
ChunkExecutionClass contracts.ExecutionClass
|
||||
ChunkCorrectionProtocol contracts.CorrectionProtocol
|
||||
ChunkReferences ResolvedReferenceTarget
|
||||
Steps []ResolvedPipelineStep
|
||||
ValidatorChains []ResolvedValidatorChain
|
||||
Output ModuleBinding
|
||||
OutputExecutionClass contracts.ExecutionClass
|
||||
ChunkControls contracts.ProducerControls
|
||||
ID string
|
||||
Input ModuleBinding
|
||||
InputExecutionClass contracts.ExecutionClass
|
||||
Chunk ModuleBinding
|
||||
ChunkExecutionClass contracts.ExecutionClass
|
||||
ChunkPrompts []string
|
||||
ChunkReferences ResolvedReferenceTarget
|
||||
Steps []ResolvedPipelineStep
|
||||
ValidatorChains []ResolvedValidatorChain
|
||||
Output ModuleBinding
|
||||
OutputExecutionClass contracts.ExecutionClass
|
||||
}{
|
||||
ID: resolved.ID,
|
||||
ChunkControls: resolved.ChunkControls,
|
||||
Input: resolved.Input,
|
||||
InputExecutionClass: resolved.InputExecutionClass,
|
||||
Chunk: resolved.Chunk,
|
||||
ChunkExecutionClass: resolved.ChunkExecutionClass,
|
||||
ChunkCorrectionProtocol: resolved.ChunkCorrectionProtocol,
|
||||
ChunkReferences: resolved.ChunkReferences,
|
||||
Steps: resolved.Steps,
|
||||
ValidatorChains: resolved.ValidatorChains,
|
||||
Output: resolved.Output,
|
||||
OutputExecutionClass: resolved.OutputExecutionClass,
|
||||
ID: resolved.ID,
|
||||
ChunkControls: resolved.ChunkControls,
|
||||
Input: resolved.Input,
|
||||
InputExecutionClass: resolved.InputExecutionClass,
|
||||
Chunk: resolved.Chunk,
|
||||
ChunkExecutionClass: resolved.ChunkExecutionClass,
|
||||
ChunkPrompts: append([]string(nil), resolved.ChunkPrompts...),
|
||||
ChunkReferences: resolved.ChunkReferences,
|
||||
Steps: resolved.Steps,
|
||||
ValidatorChains: resolved.ValidatorChains,
|
||||
Output: resolved.Output,
|
||||
OutputExecutionClass: resolved.OutputExecutionClass,
|
||||
}
|
||||
encoded, err := json.Marshal(withoutDigest)
|
||||
if err != nil {
|
||||
|
||||
@@ -22,30 +22,29 @@ type erasedMergeArtifact struct {
|
||||
}
|
||||
|
||||
type erasedTypedResult struct {
|
||||
Value any
|
||||
Diagnostics []contracts.ProducerDiagnostic
|
||||
Retry *contracts.NormalizeRetry
|
||||
ModelCandidate *contracts.ModelCandidate
|
||||
Value any
|
||||
Diagnostics []contracts.ProducerDiagnostic
|
||||
Outcome contracts.ProducerOutcome
|
||||
}
|
||||
|
||||
type typedValidationTarget struct {
|
||||
stage ModuleStage
|
||||
stepID string
|
||||
laneID string
|
||||
moduleKey string
|
||||
source *source.SourceDocument
|
||||
sourceID string
|
||||
sourceInput contracts.LLMInputMaterial
|
||||
sessionID string
|
||||
references contracts.ReferenceSet
|
||||
llmProfile string
|
||||
structuredOutputRepairAttempts *int
|
||||
metadata map[string]any
|
||||
chunk *source.Chunk
|
||||
chunks []source.Chunk
|
||||
ref source.SourceRef
|
||||
value any
|
||||
candidate *CheckpointArtifact
|
||||
stage ModuleStage
|
||||
stepID string
|
||||
laneID string
|
||||
moduleKey string
|
||||
source *source.SourceDocument
|
||||
sourceID string
|
||||
sourceInput contracts.LLMInputMaterial
|
||||
sessionID string
|
||||
references contracts.ReferenceSet
|
||||
llmProfile string
|
||||
execution contracts.ExecutionOptions
|
||||
metadata map[string]any
|
||||
chunk *source.Chunk
|
||||
chunks []source.Chunk
|
||||
ref source.SourceRef
|
||||
value any
|
||||
candidate *CheckpointArtifact
|
||||
}
|
||||
|
||||
func exactTypedValue[T any](operation string, value any) (T, error) {
|
||||
@@ -62,7 +61,18 @@ func exactTypedValue[T any](operation string, value any) (T, error) {
|
||||
return typed, nil
|
||||
}
|
||||
|
||||
type typedExtractOperation func(context.Context, any, contracts.TypedExtractionRequest) (erasedTypedResult, error)
|
||||
type typedExtractOperation func(context.Context, any, contracts.TypedExtractionRequest[any]) (erasedTypedResult, error)
|
||||
type typedMergeOperation func(context.Context, any, contracts.TypedMergeRequest[any]) (erasedTypedResult, error)
|
||||
type typedNormalizeOperation func(context.Context, any, contracts.TypedNormalizeRequest[any]) (erasedTypedResult, error)
|
||||
type typedValidateOperation func(context.Context, any, typedValidationTarget) (contracts.ValidationResult, error)
|
||||
|
||||
// specializeProducerOperation keeps type erasure inside the registry boundary.
|
||||
func specializeProducerOperation[T any](operation contracts.ProducerOperation[any]) contracts.ProducerOperation[T] {
|
||||
result := contracts.ProducerOperation[T]{Controls: operation.Controls, Execution: operation.Execution, ReuseEligible: operation.ReuseEligible}
|
||||
if operation.Assess != nil {
|
||||
result.Assess = func(ctx context.Context, value T) (contracts.CandidateAssessment, error) {
|
||||
return operation.Assess(ctx, value)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
@@ -1,253 +1,191 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
type correctionObservingNotesExtractor struct {
|
||||
correction *contracts.SemanticCorrection
|
||||
candidate *contracts.ModelCandidate
|
||||
type collaborationValue struct{ Text string }
|
||||
|
||||
type collaborationProducer struct {
|
||||
operation contracts.ProducerOperation[collaborationValue]
|
||||
outcome contracts.ProducerOutcome
|
||||
}
|
||||
|
||||
func (*correctionObservingNotesExtractor) Key() string { return "test/correction-observing-extract" }
|
||||
func (*correctionObservingNotesExtractor) ReferenceSlots() []contracts.ReferenceSlot {
|
||||
func (*collaborationProducer) Key() string { return "producer" }
|
||||
func (*collaborationProducer) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||
|
||||
func (p *collaborationProducer) assess(ctx context.Context, value collaborationValue, operation contracts.ProducerOperation[collaborationValue]) error {
|
||||
p.operation = operation
|
||||
if operation.Assess != nil {
|
||||
_, err := operation.Assess(ctx, value)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *correctionObservingNotesExtractor) Extract(_ context.Context, request contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[codecNotes], error) {
|
||||
e.correction = request.Correction
|
||||
candidate, err := contracts.NewModelCandidate([]byte(`{"items":["extract"]}`), contracts.CorrectionProtocolSingleResponseV1)
|
||||
if err != nil {
|
||||
return contracts.TypedExtractionResult[codecNotes]{}, err
|
||||
func (p *collaborationProducer) Extract(ctx context.Context, request contracts.TypedExtractionRequest[collaborationValue]) (contracts.TypedExtractionResult[collaborationValue], error) {
|
||||
value := collaborationValue{"candidate"}
|
||||
err := p.assess(ctx, value, request.Operation)
|
||||
return contracts.TypedExtractionResult[collaborationValue]{Value: value, Outcome: p.outcome}, err
|
||||
}
|
||||
func (p *collaborationProducer) Merge(ctx context.Context, request contracts.TypedMergeRequest[collaborationValue]) (contracts.TypedMergeResult[collaborationValue], error) {
|
||||
value := request.ExtractOutputs[0].Value
|
||||
err := p.assess(ctx, value, request.Operation)
|
||||
return contracts.TypedMergeResult[collaborationValue]{Value: value, Outcome: p.outcome}, err
|
||||
}
|
||||
func (p *collaborationProducer) Normalize(ctx context.Context, request contracts.TypedNormalizeRequest[collaborationValue]) (contracts.TypedNormalizeResult[collaborationValue], error) {
|
||||
value := request.MergeOutput.Value
|
||||
err := p.assess(ctx, value, request.Operation)
|
||||
return contracts.TypedNormalizeResult[collaborationValue]{Value: value, Outcome: p.outcome}, err
|
||||
}
|
||||
|
||||
type collaborationValidator struct{ execution *contracts.ExecutionOptions }
|
||||
|
||||
func (collaborationValidator) Name() string { return "assessment" }
|
||||
func (collaborationValidator) ExecutionClass() contracts.ExecutionClass {
|
||||
return contracts.ExecutionClassLLMBacked
|
||||
}
|
||||
func (v collaborationValidator) Validate(_ context.Context, request contracts.TypedValidationRequest[collaborationValue]) (contracts.ValidationResult, error) {
|
||||
if v.execution != nil {
|
||||
*v.execution = request.Execution
|
||||
}
|
||||
e.candidate = candidate
|
||||
return contracts.TypedExtractionResult[codecNotes]{Value: codecNotes{Items: []string{"extract"}}, ModelCandidate: candidate}, nil
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
}
|
||||
|
||||
type correctionObservingNotesMerger struct {
|
||||
correction *contracts.SemanticCorrection
|
||||
candidate *contracts.ModelCandidate
|
||||
}
|
||||
|
||||
func (*correctionObservingNotesMerger) Key() string { return "test/correction-observing-merge" }
|
||||
|
||||
func (m *correctionObservingNotesMerger) Merge(_ context.Context, request contracts.TypedMergeRequest[codecNotes]) (contracts.TypedMergeResult[codecNotes], error) {
|
||||
m.correction = request.Correction
|
||||
candidate, err := contracts.NewModelCandidate([]byte(`{"items":["merge"]}`), contracts.CorrectionProtocolSingleResponseV1)
|
||||
if err != nil {
|
||||
return contracts.TypedMergeResult[codecNotes]{}, err
|
||||
}
|
||||
m.candidate = candidate
|
||||
return contracts.TypedMergeResult[codecNotes]{Value: codecNotes{Items: []string{"merge"}}, ModelCandidate: candidate}, nil
|
||||
}
|
||||
|
||||
type correctionObservingNotesNormalizer struct {
|
||||
correction *contracts.SemanticCorrection
|
||||
candidate *contracts.ModelCandidate
|
||||
}
|
||||
|
||||
func (*correctionObservingNotesNormalizer) Key() string { return "test/correction-observing-normalize" }
|
||||
func (*correctionObservingNotesNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (n *correctionObservingNotesNormalizer) Normalize(_ context.Context, request contracts.TypedNormalizeRequest[codecNotes]) (contracts.TypedNormalizeResult[codecNotes], error) {
|
||||
n.correction = request.Correction
|
||||
candidate, err := contracts.NewModelCandidate([]byte(`{"items":["normalize"]}`), contracts.CorrectionProtocolSingleResponseV1)
|
||||
if err != nil {
|
||||
return contracts.TypedNormalizeResult[codecNotes]{}, err
|
||||
}
|
||||
n.candidate = candidate
|
||||
return contracts.TypedNormalizeResult[codecNotes]{Value: codecNotes{Items: []string{"normalize"}}, ModelCandidate: candidate}, nil
|
||||
}
|
||||
|
||||
type repairObservingNotesMerger struct {
|
||||
attempts *int
|
||||
}
|
||||
|
||||
func (*repairObservingNotesMerger) Key() string { return "test/repair-observing-merge" }
|
||||
|
||||
func (m *repairObservingNotesMerger) Merge(_ context.Context, request contracts.TypedMergeRequest[codecNotes]) (contracts.TypedMergeResult[codecNotes], error) {
|
||||
m.attempts = request.StructuredOutputRepairAttempts
|
||||
return contracts.TypedMergeResult[codecNotes]{Value: codecNotes{Items: []string{"merged"}}}, nil
|
||||
}
|
||||
|
||||
type repairObservingNotesNormalizer struct {
|
||||
attempts *int
|
||||
}
|
||||
|
||||
func (*repairObservingNotesNormalizer) Key() string { return "test/repair-observing-normalize" }
|
||||
func (*repairObservingNotesNormalizer) ReferenceSlots() []contracts.ReferenceSlot {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (n *repairObservingNotesNormalizer) Normalize(_ context.Context, request contracts.TypedNormalizeRequest[codecNotes]) (contracts.TypedNormalizeResult[codecNotes], error) {
|
||||
n.attempts = request.StructuredOutputRepairAttempts
|
||||
return contracts.TypedNormalizeResult[codecNotes]{Value: request.MergeOutput.Value}, nil
|
||||
}
|
||||
|
||||
func TestTypedRegistryErasurePreservesStructuredOutputRepairAttempts(t *testing.T) {
|
||||
const artifactKind contracts.ArtifactKind = "test/notes"
|
||||
|
||||
t.Run("merge", func(t *testing.T) {
|
||||
implementation := &repairObservingNotesMerger{}
|
||||
registry := NewMergerRegistry()
|
||||
if err := RegisterMerger(registry, ModuleSpec{Key: implementation.Key(), Stage: StageMerge, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: artifactKind}, func() (contracts.Merger[codecNotes], error) {
|
||||
return implementation, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("RegisterMerger() error = %v", err)
|
||||
}
|
||||
entry, ok := registry.typedEntry(implementation.Key(), artifactKind)
|
||||
if !ok {
|
||||
t.Fatal("typed merger entry missing")
|
||||
}
|
||||
built, err := entry.builder(BuildRequest{})
|
||||
if err != nil {
|
||||
t.Fatalf("builder() error = %v", err)
|
||||
}
|
||||
attempts := 2
|
||||
_, err = entry.merge(context.Background(), built, contracts.TypedMergeRequest[any]{
|
||||
ExtractOutputs: []contracts.ExtractArtifact[any]{{Value: codecNotes{Items: []string{"extracted"}}}},
|
||||
StructuredOutputRepairAttempts: &attempts,
|
||||
func TestTypedRegistryCarriesOperationAssessmentAndOutcome(t *testing.T) {
|
||||
for _, stage := range []ModuleStage{StageExtract, StageMerge, StageNormalize} {
|
||||
t.Run(string(stage), func(t *testing.T) {
|
||||
expected := collaborationValue{"candidate"}
|
||||
calls := 0
|
||||
zero := 0
|
||||
native := &promptkit.ExecutionTargetOverride{}
|
||||
operation := contracts.ProducerOperation[any]{
|
||||
Controls: contracts.ProducerControls{Mode: contracts.ModePermissive, MaxAdditionalTurns: 0, Cache: contracts.CacheBypass},
|
||||
Execution: contracts.ExecutionOptions{Execution: native, TransientRetries: &zero},
|
||||
ReuseEligible: false,
|
||||
Assess: func(_ context.Context, value any) (contracts.CandidateAssessment, error) {
|
||||
calls++
|
||||
if reflect.TypeOf(value) != reflect.TypeFor[collaborationValue]() || value != expected {
|
||||
t.Fatalf("candidate = %#v", value)
|
||||
}
|
||||
return contracts.CandidateAssessment{}, nil
|
||||
},
|
||||
}
|
||||
impl := &collaborationProducer{outcome: contracts.ProducerOutcome{Disposition: contracts.ProducerFallback, FallbackReason: "rejected", ReuseEligible: false}}
|
||||
invoke := registerCollaborationProducer(t, stage, impl)
|
||||
result, err := invoke(context.Background(), operation, expected)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if calls != 1 || result.Value != expected || result.Outcome != impl.outcome {
|
||||
t.Fatalf("result = %#v, assessment calls = %d", result, calls)
|
||||
}
|
||||
got := impl.operation
|
||||
if got.Controls != operation.Controls || got.ReuseEligible || got.Execution.Execution != native || got.Execution.TransientRetries == nil || *got.Execution.TransientRetries != 0 {
|
||||
t.Fatalf("operation lost at type boundary: %#v", got)
|
||||
}
|
||||
sentinel := errors.New("assessment contract failure")
|
||||
operation.Assess = func(context.Context, any) (contracts.CandidateAssessment, error) {
|
||||
return contracts.CandidateAssessment{}, sentinel
|
||||
}
|
||||
if _, err := invoke(context.Background(), operation, expected); !errors.Is(err, sentinel) {
|
||||
t.Fatalf("assessment error = %v", err)
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("merge() error = %v", err)
|
||||
}
|
||||
if implementation.attempts == nil || *implementation.attempts != attempts {
|
||||
t.Fatalf("repair attempts = %v, want %d", implementation.attempts, attempts)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("normalize", func(t *testing.T) {
|
||||
implementation := &repairObservingNotesNormalizer{}
|
||||
registry := NewNormalizerRegistry()
|
||||
if err := RegisterNormalizer(registry, ModuleSpec{Key: implementation.Key(), Stage: StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: artifactKind}, func() (contracts.Normalizer[codecNotes], error) {
|
||||
return implementation, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("RegisterNormalizer() error = %v", err)
|
||||
}
|
||||
entry, ok := registry.typedEntry(implementation.Key(), artifactKind)
|
||||
if !ok {
|
||||
t.Fatal("typed normalizer entry missing")
|
||||
}
|
||||
built, err := entry.builder(BuildRequest{})
|
||||
if err != nil {
|
||||
t.Fatalf("builder() error = %v", err)
|
||||
}
|
||||
attempts := 3
|
||||
_, err = entry.normalize(context.Background(), built, contracts.TypedNormalizeRequest[any]{
|
||||
MergeOutput: contracts.MergeArtifact[any]{Value: codecNotes{Items: []string{"merged"}}},
|
||||
StructuredOutputRepairAttempts: &attempts,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("normalize() error = %v", err)
|
||||
}
|
||||
if implementation.attempts == nil || *implementation.attempts != attempts {
|
||||
t.Fatalf("repair attempts = %v, want %d", implementation.attempts, attempts)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTypedRegistryErasurePreservesCorrectionAndCandidateOwnership(t *testing.T) {
|
||||
const artifactKind contracts.ArtifactKind = "test/notes"
|
||||
|
||||
t.Run("extract", func(t *testing.T) {
|
||||
correction := newTestCorrection(t)
|
||||
implementation := &correctionObservingNotesExtractor{}
|
||||
func registerCollaborationProducer(t *testing.T, stage ModuleStage, impl *collaborationProducer) func(context.Context, contracts.ProducerOperation[any], any) (erasedTypedResult, error) {
|
||||
t.Helper()
|
||||
spec := ModuleSpec{Key: impl.Key(), Stage: stage, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: "test/value"}
|
||||
switch stage {
|
||||
case StageExtract:
|
||||
registry := NewExtractorRegistry()
|
||||
if err := RegisterExtractor(registry, ModuleSpec{Key: implementation.Key(), Stage: StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: artifactKind}, func() (contracts.Extractor[codecNotes], error) {
|
||||
return implementation, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("RegisterExtractor() error = %v", err)
|
||||
if err := RegisterExtractor(registry, spec, func() (contracts.Extractor[collaborationValue], error) { return impl, nil }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
entry, ok := registry.typedEntry(implementation.Key())
|
||||
if !ok {
|
||||
t.Fatal("typed extractor entry missing")
|
||||
entry, _ := registry.typedEntry(spec.Key)
|
||||
return func(ctx context.Context, op contracts.ProducerOperation[any], _ any) (erasedTypedResult, error) {
|
||||
return entry.extract(ctx, impl, contracts.TypedExtractionRequest[any]{Operation: op})
|
||||
}
|
||||
built, err := entry.builder(BuildRequest{})
|
||||
if err != nil {
|
||||
t.Fatalf("builder() error = %v", err)
|
||||
}
|
||||
result, err := entry.extract(context.Background(), built, contracts.TypedExtractionRequest{Correction: correction})
|
||||
if err != nil {
|
||||
t.Fatalf("extract() error = %v", err)
|
||||
}
|
||||
assertCorrectionAndCandidateOwnership(t, correction, implementation.correction, implementation.candidate, result.ModelCandidate)
|
||||
})
|
||||
|
||||
t.Run("merge", func(t *testing.T) {
|
||||
correction := newTestCorrection(t)
|
||||
implementation := &correctionObservingNotesMerger{}
|
||||
case StageMerge:
|
||||
registry := NewMergerRegistry()
|
||||
if err := RegisterMerger(registry, ModuleSpec{Key: implementation.Key(), Stage: StageMerge, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: artifactKind}, func() (contracts.Merger[codecNotes], error) {
|
||||
return implementation, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("RegisterMerger() error = %v", err)
|
||||
if err := RegisterMerger(registry, spec, func() (contracts.Merger[collaborationValue], error) { return impl, nil }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
entry, ok := registry.typedEntry(implementation.Key(), artifactKind)
|
||||
if !ok {
|
||||
t.Fatal("typed merger entry missing")
|
||||
entry, _ := registry.typedEntry(spec.Key, spec.ArtifactKind)
|
||||
return func(ctx context.Context, op contracts.ProducerOperation[any], value any) (erasedTypedResult, error) {
|
||||
return entry.merge(ctx, impl, contracts.TypedMergeRequest[any]{Operation: op, ExtractOutputs: []contracts.ExtractArtifact[any]{{Value: value}}})
|
||||
}
|
||||
built, err := entry.builder(BuildRequest{})
|
||||
if err != nil {
|
||||
t.Fatalf("builder() error = %v", err)
|
||||
}
|
||||
result, err := entry.merge(context.Background(), built, contracts.TypedMergeRequest[any]{Correction: correction, ExtractOutputs: []contracts.ExtractArtifact[any]{{Value: codecNotes{Items: []string{"extract"}}}}})
|
||||
if err != nil {
|
||||
t.Fatalf("merge() error = %v", err)
|
||||
}
|
||||
assertCorrectionAndCandidateOwnership(t, correction, implementation.correction, implementation.candidate, result.ModelCandidate)
|
||||
})
|
||||
|
||||
t.Run("normalize", func(t *testing.T) {
|
||||
correction := newTestCorrection(t)
|
||||
implementation := &correctionObservingNotesNormalizer{}
|
||||
default:
|
||||
registry := NewNormalizerRegistry()
|
||||
if err := RegisterNormalizer(registry, ModuleSpec{Key: implementation.Key(), Stage: StageNormalize, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: artifactKind}, func() (contracts.Normalizer[codecNotes], error) {
|
||||
return implementation, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("RegisterNormalizer() error = %v", err)
|
||||
if err := RegisterNormalizer(registry, spec, func() (contracts.Normalizer[collaborationValue], error) { return impl, nil }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
entry, ok := registry.typedEntry(implementation.Key(), artifactKind)
|
||||
if !ok {
|
||||
t.Fatal("typed normalizer entry missing")
|
||||
entry, _ := registry.typedEntry(spec.Key, spec.ArtifactKind)
|
||||
return func(ctx context.Context, op contracts.ProducerOperation[any], value any) (erasedTypedResult, error) {
|
||||
return entry.normalize(ctx, impl, contracts.TypedNormalizeRequest[any]{Operation: op, MergeOutput: contracts.MergeArtifact[any]{Value: value}})
|
||||
}
|
||||
built, err := entry.builder(BuildRequest{})
|
||||
if err != nil {
|
||||
t.Fatalf("builder() error = %v", err)
|
||||
}
|
||||
result, err := entry.normalize(context.Background(), built, contracts.TypedNormalizeRequest[any]{Correction: correction, MergeOutput: contracts.MergeArtifact[any]{Value: codecNotes{Items: []string{"merge"}}}})
|
||||
if err != nil {
|
||||
t.Fatalf("normalize() error = %v", err)
|
||||
}
|
||||
assertCorrectionAndCandidateOwnership(t, correction, implementation.correction, implementation.candidate, result.ModelCandidate)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func newTestCorrection(t *testing.T) *contracts.SemanticCorrection {
|
||||
t.Helper()
|
||||
correction, err := contracts.NewSemanticCorrection([]byte(`{"items":["original"]}`), "Correct the response.")
|
||||
if err != nil {
|
||||
t.Fatalf("NewSemanticCorrection() error = %v", err)
|
||||
func TestTypedRegistryRejectsIncompatibleOperationValues(t *testing.T) {
|
||||
for _, stage := range []ModuleStage{StageMerge, StageNormalize} {
|
||||
invoke := registerCollaborationProducer(t, stage, &collaborationProducer{})
|
||||
if _, err := invoke(context.Background(), contracts.ProducerOperation[any]{}, struct{ Text string }{"candidate"}); err == nil {
|
||||
t.Fatalf("%s accepted structurally similar but incompatible Go type", stage)
|
||||
}
|
||||
}
|
||||
return correction
|
||||
}
|
||||
|
||||
func assertCorrectionAndCandidateOwnership(t *testing.T, callerCorrection, observedCorrection *contracts.SemanticCorrection, producerCandidate, returnedCandidate *contracts.ModelCandidate) {
|
||||
t.Helper()
|
||||
if observedCorrection == nil || producerCandidate == nil || returnedCandidate == nil {
|
||||
t.Fatal("correction and candidates must be present")
|
||||
func TestConstructionPassesTheSharedNativeEngine(t *testing.T) {
|
||||
engine := &promptkit.Engine{}
|
||||
registry := NewExtractorRegistry()
|
||||
spec := ModuleSpec{Key: "producer", Stage: StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: "test/value"}
|
||||
if err := RegisterExtractorBuilder(registry, spec, rejectUnconfiguredOptions, func(request BuildRequest) (contracts.Extractor[collaborationValue], error) {
|
||||
if request.Dependencies.Engine != engine {
|
||||
t.Fatal("builder received a different engine")
|
||||
}
|
||||
return &collaborationProducer{}, nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
callerCorrection.AssistantResponse[0] = '['
|
||||
if got := string(observedCorrection.AssistantResponse); got != `{"items":["original"]}` {
|
||||
t.Fatalf("observed correction = %q, want owned original content", got)
|
||||
}
|
||||
producerCandidate.Response[0] = '['
|
||||
if bytes.Equal(producerCandidate.Response, returnedCandidate.Response) {
|
||||
t.Fatalf("returned candidate aliases producer candidate: %q", returnedCandidate.Response)
|
||||
entry, _ := registry.typedEntry(spec.Key)
|
||||
if _, err := entry.builder(BuildRequest{Dependencies: ModuleDependencies{Engine: engine}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTypedValidatorReceivesNativeExecutionOptions(t *testing.T) {
|
||||
var got contracts.ExecutionOptions
|
||||
implementation := collaborationValidator{execution: &got}
|
||||
registry := NewValidatorRegistry()
|
||||
if err := RegisterTypedValidator(registry, "test/value", ValidatorSpec{Key: "assessment", ExecutionClass: contracts.ExecutionClassLLMBacked}, func() (contracts.TypedValidator[collaborationValue], error) { return implementation, nil }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
entry, _ := registry.typedEntry("assessment", "test/value")
|
||||
observed, prepared := false, false
|
||||
native := &promptkit.ExecutionTargetOverride{}
|
||||
zero := 0
|
||||
options := contracts.ExecutionOptions{Execution: native, TransientRetries: &zero,
|
||||
Observer: func(promptkit.ExecutionEvent) { observed = true },
|
||||
OnPrepared: func(context.Context, promptkit.PreparedRun) { prepared = true },
|
||||
}
|
||||
if _, err := entry.validate(context.Background(), implementation, typedValidationTarget{value: collaborationValue{"candidate"}, execution: options}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.Execution != native || got.TransientRetries == nil || *got.TransientRetries != 0 || got.Observer == nil || got.OnPrepared == nil {
|
||||
t.Fatalf("native options lost: %#v", got)
|
||||
}
|
||||
got.Observer(promptkit.ExecutionEvent{})
|
||||
got.OnPrepared(context.Background(), promptkit.PreparedRun{})
|
||||
if !observed || !prepared {
|
||||
t.Fatal("native observation callbacks were not retained")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,7 +15,7 @@ type typedTestExtractor[T any] struct{ key string }
|
||||
|
||||
func (e typedTestExtractor[T]) Key() string { return e.key }
|
||||
func (typedTestExtractor[T]) ReferenceSlots() []contracts.ReferenceSlot { return nil }
|
||||
func (typedTestExtractor[T]) Extract(context.Context, contracts.TypedExtractionRequest) (contracts.TypedExtractionResult[T], error) {
|
||||
func (typedTestExtractor[T]) Extract(context.Context, contracts.TypedExtractionRequest[T]) (contracts.TypedExtractionResult[T], error) {
|
||||
return contracts.TypedExtractionResult[T]{}, nil
|
||||
}
|
||||
|
||||
@@ -586,3 +586,34 @@ var _ contracts.Normalizer[codecNotes] = typedTestNormalizer[codecNotes]{}
|
||||
var _ contracts.TypedValidator[codecNotes] = typedTestValidator[codecNotes]{}
|
||||
var _ contracts.ChunkValidator = typedTestChunkValidator{}
|
||||
var _ contracts.SerializedValidator = typedTestSerializedValidator{}
|
||||
|
||||
func TestPrepareRetainsInitialAndCorrectionDependencies(t *testing.T) {
|
||||
catalog := typedResolutionCatalog(t, completeTypedCatalogOptions())
|
||||
const initial = "notarius/test.initial"
|
||||
const correction = "notarius/test.correction"
|
||||
const key = "typed/native-notes"
|
||||
prompts := []string{initial, correction}
|
||||
if err := RegisterExtractor(catalog.Extractors, ModuleSpec{Key: key, Stage: StageExtract, ExecutionClass: contracts.ExecutionClassLLMBacked, ArtifactKind: "test/notes", Prompts: prompts}, func() (contracts.Extractor[codecNotes], error) { return typedTestExtractor[codecNotes]{key: key}, nil }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
profile := typedResolutionProfile()
|
||||
lane := profile.Artifacts["notes"]
|
||||
lane.Extract.Module = key
|
||||
profile.Artifacts["notes"] = lane
|
||||
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
prepared, err := Prepare(resolved, registriesFromModuleCatalog(catalog), ModuleDependencies{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []string{correction, initial}
|
||||
if got := prepared.Steps[0].ArtifactLanes[0].ExtractPrompts; !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("prepared dependencies = %#v", got)
|
||||
}
|
||||
resolved.Steps[0].ArtifactLanes[0].ExtractPrompts[0] = "changed"
|
||||
if got := prepared.Steps[0].ArtifactLanes[0].Resolved.ExtractPrompts; !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("preparation retained mutable dependencies = %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,54 +3,34 @@ package pipeline
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
type ExecutionMode string
|
||||
|
||||
const (
|
||||
ModeStrict ExecutionMode = "strict"
|
||||
ModePermissive ExecutionMode = "permissive"
|
||||
)
|
||||
|
||||
func ParseExecutionMode(value string) (ExecutionMode, error) {
|
||||
mode := ExecutionMode(strings.TrimSpace(value))
|
||||
if mode != ModeStrict && mode != ModePermissive {
|
||||
func ParseExecutionMode(value string) (contracts.ExecutionMode, error) {
|
||||
mode := contracts.ExecutionMode(strings.TrimSpace(value))
|
||||
if mode != contracts.ModeStrict && mode != contracts.ModePermissive {
|
||||
return "", fmt.Errorf("mode must be strict or permissive")
|
||||
}
|
||||
return mode, nil
|
||||
}
|
||||
|
||||
type CacheMode string
|
||||
|
||||
const (
|
||||
CacheAuto CacheMode = "auto"
|
||||
CacheBypass CacheMode = "bypass"
|
||||
CacheRefresh CacheMode = "refresh"
|
||||
)
|
||||
|
||||
func ParseCacheMode(value string) (CacheMode, error) {
|
||||
mode := CacheMode(strings.TrimSpace(value))
|
||||
func ParseCacheMode(value string) (contracts.CacheMode, error) {
|
||||
mode := contracts.CacheMode(strings.TrimSpace(value))
|
||||
switch mode {
|
||||
case CacheAuto, CacheBypass, CacheRefresh:
|
||||
case contracts.CacheAuto, contracts.CacheBypass, contracts.CacheRefresh:
|
||||
return mode, nil
|
||||
}
|
||||
return "", fmt.Errorf("cache must be auto, bypass, or refresh")
|
||||
}
|
||||
|
||||
// ProducerControls contains final operation controls; execution does not inherit settings.
|
||||
type ProducerControls struct {
|
||||
Mode ExecutionMode `json:"mode"`
|
||||
MaxAdditionalTurns int `json:"max_additional_turns"`
|
||||
Cache CacheMode `json:"cache"`
|
||||
}
|
||||
|
||||
func ResolveProducerControls(binding ModuleBinding, profile PipelineProfile, invocationCache *CacheMode) (ProducerControls, error) {
|
||||
result := ProducerControls{Mode: ModeStrict, MaxAdditionalTurns: 2, Cache: CacheAuto}
|
||||
for _, mode := range []*ExecutionMode{profile.Mode, binding.Mode} {
|
||||
func ResolveProducerControls(binding ModuleBinding, profile PipelineProfile, invocationCache *contracts.CacheMode) (contracts.ProducerControls, error) {
|
||||
result := contracts.ProducerControls{Mode: contracts.ModeStrict, MaxAdditionalTurns: 2, Cache: contracts.CacheAuto}
|
||||
for _, mode := range []*contracts.ExecutionMode{profile.Mode, binding.Mode} {
|
||||
if mode != nil {
|
||||
parsed, err := ParseExecutionMode(string(*mode))
|
||||
if err != nil {
|
||||
return ProducerControls{}, err
|
||||
return contracts.ProducerControls{}, err
|
||||
}
|
||||
result.Mode = parsed
|
||||
}
|
||||
@@ -58,16 +38,16 @@ func ResolveProducerControls(binding ModuleBinding, profile PipelineProfile, inv
|
||||
for _, turns := range []*int{profile.MaxAdditionalTurns, binding.MaxAdditionalTurns} {
|
||||
if turns != nil {
|
||||
if *turns < 0 {
|
||||
return ProducerControls{}, fmt.Errorf("max_additional_turns must not be negative")
|
||||
return contracts.ProducerControls{}, fmt.Errorf("max_additional_turns must not be negative")
|
||||
}
|
||||
result.MaxAdditionalTurns = *turns
|
||||
}
|
||||
}
|
||||
for _, cache := range []*CacheMode{profile.Cache, binding.Cache, invocationCache} {
|
||||
for _, cache := range []*contracts.CacheMode{profile.Cache, binding.Cache, invocationCache} {
|
||||
if cache != nil {
|
||||
parsed, err := ParseCacheMode(string(*cache))
|
||||
if err != nil {
|
||||
return ProducerControls{}, err
|
||||
return contracts.ProducerControls{}, err
|
||||
}
|
||||
result.Cache = parsed
|
||||
}
|
||||
@@ -83,11 +63,11 @@ func cloneControl[T any](value *T) *T {
|
||||
return ©
|
||||
}
|
||||
|
||||
func applyEffectiveProducerControls(resolved *ResolvedPipeline, profile PipelineProfile, invocationCache *CacheMode) error {
|
||||
apply := func(binding ModuleBinding, scope string) (ProducerControls, error) {
|
||||
func applyEffectiveProducerControls(resolved *ResolvedPipeline, profile PipelineProfile, invocationCache *contracts.CacheMode) error {
|
||||
apply := func(binding ModuleBinding, scope string) (contracts.ProducerControls, error) {
|
||||
controls, err := ResolveProducerControls(binding, profile, invocationCache)
|
||||
if err != nil {
|
||||
return ProducerControls{}, fmt.Errorf("pipeline %q %s: %w", resolved.ID, scope, err)
|
||||
return contracts.ProducerControls{}, fmt.Errorf("pipeline %q %s: %w", resolved.ID, scope, err)
|
||||
}
|
||||
return controls, nil
|
||||
}
|
||||
|
||||
@@ -3,23 +3,25 @@ package pipeline
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
func TestProducerControlsResolvePrecedenceAndExplicitZero(t *testing.T) {
|
||||
strict, permissive := ModeStrict, ModePermissive
|
||||
auto, bypass, refresh := CacheAuto, CacheBypass, CacheRefresh
|
||||
strict, permissive := contracts.ModeStrict, contracts.ModePermissive
|
||||
auto, bypass, refresh := contracts.CacheAuto, contracts.CacheBypass, contracts.CacheRefresh
|
||||
zero, five := 0, 5
|
||||
tests := []struct {
|
||||
name string
|
||||
profile PipelineProfile
|
||||
binding ModuleBinding
|
||||
invocation *CacheMode
|
||||
want ProducerControls
|
||||
invocation *contracts.CacheMode
|
||||
want contracts.ProducerControls
|
||||
}{
|
||||
{"defaults", PipelineProfile{}, ModuleBinding{}, nil, ProducerControls{ModeStrict, 2, CacheAuto}},
|
||||
{"pipeline", PipelineProfile{Mode: &permissive, MaxAdditionalTurns: &five, Cache: &refresh}, ModuleBinding{}, nil, ProducerControls{ModePermissive, 5, CacheRefresh}},
|
||||
{"binding", PipelineProfile{Mode: &permissive, MaxAdditionalTurns: &five, Cache: &refresh}, ModuleBinding{Mode: &strict, MaxAdditionalTurns: &zero, Cache: &auto}, nil, ProducerControls{ModeStrict, 0, CacheAuto}},
|
||||
{"invocation", PipelineProfile{Cache: &refresh}, ModuleBinding{Cache: &auto}, &bypass, ProducerControls{ModeStrict, 2, CacheBypass}},
|
||||
{"defaults", PipelineProfile{}, ModuleBinding{}, nil, contracts.ProducerControls{contracts.ModeStrict, 2, contracts.CacheAuto}},
|
||||
{"pipeline", PipelineProfile{Mode: &permissive, MaxAdditionalTurns: &five, Cache: &refresh}, ModuleBinding{}, nil, contracts.ProducerControls{contracts.ModePermissive, 5, contracts.CacheRefresh}},
|
||||
{"binding", PipelineProfile{Mode: &permissive, MaxAdditionalTurns: &five, Cache: &refresh}, ModuleBinding{Mode: &strict, MaxAdditionalTurns: &zero, Cache: &auto}, nil, contracts.ProducerControls{contracts.ModeStrict, 0, contracts.CacheAuto}},
|
||||
{"invocation", PipelineProfile{Cache: &refresh}, ModuleBinding{Cache: &auto}, &bypass, contracts.ProducerControls{contracts.ModeStrict, 2, contracts.CacheBypass}},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
@@ -35,7 +37,7 @@ func TestProducerControlsResolvePrecedenceAndExplicitZero(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestProducerControlsRejectInvalidValuesWithoutMaximumBudgets(t *testing.T) {
|
||||
invalidMode, invalidCache := ExecutionMode("unknown"), CacheMode("unknown")
|
||||
invalidMode, invalidCache := contracts.ExecutionMode("unknown"), contracts.CacheMode("unknown")
|
||||
negative, large := -1, 1000000
|
||||
for _, binding := range []ModuleBinding{{Mode: &invalidMode}, {Cache: &invalidCache}, {MaxAdditionalTurns: &negative}} {
|
||||
if _, err := ResolveProducerControls(binding, PipelineProfile{}, nil); err == nil {
|
||||
@@ -49,7 +51,7 @@ func TestProducerControlsRejectInvalidValuesWithoutMaximumBudgets(t *testing.T)
|
||||
}
|
||||
|
||||
func TestModuleBindingRoundTripRetainsExplicitControls(t *testing.T) {
|
||||
strict, bypass, zero := ModeStrict, CacheBypass, 0
|
||||
strict, bypass, zero := contracts.ModeStrict, contracts.CacheBypass, 0
|
||||
input := ModuleBinding{Module: "fixture", Mode: &strict, Cache: &bypass, MaxAdditionalTurns: &zero}
|
||||
data, err := json.Marshal(input)
|
||||
if err != nil {
|
||||
@@ -66,20 +68,20 @@ func TestModuleBindingRoundTripRetainsExplicitControls(t *testing.T) {
|
||||
|
||||
func TestResolvedProducerControlsApplyToEveryLaneOperation(t *testing.T) {
|
||||
profile := baselineProfile()
|
||||
permissive, bypass, zero := ModePermissive, CacheBypass, 0
|
||||
permissive, bypass, zero := contracts.ModePermissive, contracts.CacheBypass, 0
|
||||
profile.Mode = &permissive
|
||||
profile.Chunk.MaxAdditionalTurns = &zero
|
||||
resolved, err := ResolvePipeline(profile, ResolveOptions{CacheOverride: &bypass}, newProfileCatalog(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if resolved.ChunkControls != (ProducerControls{ModePermissive, 0, CacheBypass}) {
|
||||
if resolved.ChunkControls != (contracts.ProducerControls{contracts.ModePermissive, 0, contracts.CacheBypass}) {
|
||||
t.Fatalf("chunk controls = %#v", resolved.ChunkControls)
|
||||
}
|
||||
for _, step := range resolved.Steps {
|
||||
for _, lane := range step.ArtifactLanes {
|
||||
for _, got := range []ProducerControls{lane.ExtractControls, lane.MergeControls, lane.NormalizeControls} {
|
||||
if got != (ProducerControls{ModePermissive, 2, CacheBypass}) {
|
||||
for _, got := range []contracts.ProducerControls{lane.ExtractControls, lane.MergeControls, lane.NormalizeControls} {
|
||||
if got != (contracts.ProducerControls{contracts.ModePermissive, 2, contracts.CacheBypass}) {
|
||||
t.Fatalf("lane controls = %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,9 +11,10 @@ import (
|
||||
)
|
||||
|
||||
type ValidatorSpec struct {
|
||||
Key string `json:"key"`
|
||||
ExecutionClass contracts.ExecutionClass `json:"execution_class"`
|
||||
CorrectionProtocol contracts.CorrectionProtocol `json:"correction_protocol,omitempty"`
|
||||
Key string `json:"key"`
|
||||
ExecutionClass contracts.ExecutionClass `json:"execution_class"`
|
||||
// Prompts identifies assessment requests, not a producer correction loop.
|
||||
Prompts []string `json:"prompts,omitempty"`
|
||||
}
|
||||
|
||||
type SerializedValidatorSpec struct {
|
||||
@@ -116,7 +117,7 @@ func RegisterTypedValidatorBuilder[T any](registry *ValidatorRegistry, kind cont
|
||||
if err != nil {
|
||||
return contracts.ValidationResult{}, err
|
||||
}
|
||||
return validator.Validate(ctx, contracts.TypedValidationRequest[T]{Stage: string(target.stage), LaneID: target.laneID, ModuleKey: target.moduleKey, Source: target.source, SourceID: target.sourceID, SourceInput: target.sourceInput, SessionID: target.sessionID, References: target.references, LLMProfile: target.llmProfile, StructuredOutputRepairAttempts: target.structuredOutputRepairAttempts, Metadata: target.metadata, Chunk: target.chunk, Chunks: target.chunks, Ref: target.ref, Value: value})
|
||||
return validator.Validate(ctx, contracts.TypedValidationRequest[T]{Stage: string(target.stage), LaneID: target.laneID, ModuleKey: target.moduleKey, Source: target.source, SourceID: target.sourceID, SourceInput: target.sourceInput, SessionID: target.sessionID, References: target.references, LLMProfile: target.llmProfile, Execution: target.execution, Metadata: target.metadata, Chunk: target.chunk, Chunks: target.chunks, Ref: target.ref, Value: value})
|
||||
},
|
||||
}
|
||||
return nil
|
||||
@@ -239,14 +240,14 @@ func (r *ValidatorRegistry) Spec(key string) (ValidatorSpec, bool) {
|
||||
}
|
||||
normalized := strings.TrimSpace(key)
|
||||
if entry, found := r.chunkEntries[normalized]; found {
|
||||
return entry.spec, true
|
||||
return cloneValidatorSpec(entry.spec), true
|
||||
}
|
||||
if entry, found := r.serializedEntries[normalized]; found {
|
||||
return entry.spec.ValidatorSpec, true
|
||||
return cloneValidatorSpec(entry.spec.ValidatorSpec), true
|
||||
}
|
||||
if kinds := r.registeredTypedKinds(normalized); len(kinds) > 0 {
|
||||
entry, found := r.typedEntry(normalized, kinds[0])
|
||||
return entry.spec, found
|
||||
return cloneValidatorSpec(entry.spec), found
|
||||
}
|
||||
return ValidatorSpec{}, false
|
||||
}
|
||||
@@ -323,9 +324,9 @@ func (r *ValidatorRegistry) RegisteredKeys() []string {
|
||||
|
||||
func normalizeValidatorSpec(spec ValidatorSpec) (ValidatorSpec, error) {
|
||||
normalized := ValidatorSpec{
|
||||
Key: strings.TrimSpace(spec.Key),
|
||||
ExecutionClass: spec.ExecutionClass,
|
||||
CorrectionProtocol: contracts.CorrectionProtocol(strings.TrimSpace(string(spec.CorrectionProtocol))),
|
||||
Key: strings.TrimSpace(spec.Key),
|
||||
ExecutionClass: spec.ExecutionClass,
|
||||
Prompts: normalizeStringSet(spec.Prompts),
|
||||
}
|
||||
if normalized.Key == "" {
|
||||
return ValidatorSpec{}, fmt.Errorf("validator key must not be empty")
|
||||
@@ -335,12 +336,14 @@ func normalizeValidatorSpec(spec ValidatorSpec) (ValidatorSpec, error) {
|
||||
default:
|
||||
return ValidatorSpec{}, fmt.Errorf("validator %q execution class %q is not supported", normalized.Key, normalized.ExecutionClass)
|
||||
}
|
||||
if normalized.CorrectionProtocol != "" {
|
||||
return ValidatorSpec{}, fmt.Errorf("validator %q correction protocol is not supported", normalized.Key)
|
||||
}
|
||||
return normalized, nil
|
||||
}
|
||||
|
||||
func sortValidatorSpecs(specs []ValidatorSpec) {
|
||||
sort.Slice(specs, func(i, j int) bool { return specs[i].Key < specs[j].Key })
|
||||
}
|
||||
|
||||
func cloneValidatorSpec(spec ValidatorSpec) ValidatorSpec {
|
||||
spec.Prompts = append([]string(nil), spec.Prompts...)
|
||||
return spec
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user