Add offline pipeline inspection and shared native resource preflight
This commit is contained in:
120
internal/preflight/preflight.go
Normal file
120
internal/preflight/preflight.go
Normal file
@@ -0,0 +1,120 @@
|
||||
// Package preflight checks statically known native resources for an already
|
||||
// resolved pipeline. Acquisition, construction, and presentation belong to callers.
|
||||
package preflight
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
// RequiresEngine reports selected model work, including nested validators.
|
||||
// Deterministic pipelines need no engine merely to validate their composition.
|
||||
func RequiresEngine(resolved pipeline.ResolvedPipeline) bool {
|
||||
if resolved.InputExecutionClass == contracts.ExecutionClassLLMBacked || resolved.ChunkExecutionClass == contracts.ExecutionClassLLMBacked || resolved.OutputExecutionClass == contracts.ExecutionClassLLMBacked {
|
||||
return true
|
||||
}
|
||||
for _, step := range resolved.Steps {
|
||||
for _, lane := range step.ArtifactLanes {
|
||||
if lane.ExtractExecutionClass == contracts.ExecutionClassLLMBacked || lane.MergeExecutionClass == contracts.ExecutionClassLLMBacked || lane.NormalizeExecutionClass == contracts.ExecutionClassLLMBacked {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, chain := range resolved.ValidatorChains {
|
||||
for _, validator := range chain.Validators {
|
||||
if validator.ExecutionClass == contracts.ExecutionClassLLMBacked {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Validate checks the same declarations and resource selections used by native
|
||||
// request builders. It does not render, prepare, read input/reference files,
|
||||
// resolve credential values, or call providers. Dynamic selections remain runtime
|
||||
// checks. Callers first use their ordinary application resolver for module,
|
||||
// options, type, validator-chain, and ordered-reference compatibility.
|
||||
func Validate(ctx context.Context, resolved pipeline.ResolvedPipeline, catalog pipeline.ModuleCatalog, engine *promptkit.Engine) error {
|
||||
if ctx == nil {
|
||||
return errors.New("preflight context must not be nil")
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
var findings []error
|
||||
check := func(stage pipeline.ModuleStage, step, lane, module, validator string, class contracts.ExecutionClass, profile string, prompts []string) {
|
||||
if class != contracts.ExecutionClassLLMBacked || ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
scope := fmt.Sprintf("pipeline %q", resolved.ID)
|
||||
if step != "" {
|
||||
scope += fmt.Sprintf(" step %q", step)
|
||||
}
|
||||
if lane != "" {
|
||||
scope += fmt.Sprintf(" lane %q", lane)
|
||||
}
|
||||
scope += fmt.Sprintf(" %s module %q", stage, module)
|
||||
if validator != "" {
|
||||
scope += fmt.Sprintf(" validator %q", validator)
|
||||
}
|
||||
if engine == nil {
|
||||
findings = append(findings, fmt.Errorf("%s: %w: native engine is required", scope, promptkit.ErrInvalidConfig))
|
||||
return
|
||||
}
|
||||
if profile != "" {
|
||||
if _, err := engine.InspectProfile(ctx, profile); err != nil {
|
||||
findings = append(findings, fmt.Errorf("%s profile %q: %w", scope, profile, err))
|
||||
}
|
||||
}
|
||||
for _, id := range prompts {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
prompt, err := engine.InspectPrompt(ctx, id)
|
||||
if err != nil {
|
||||
findings = append(findings, fmt.Errorf("%s prompt %q: %w", scope, id, err))
|
||||
continue
|
||||
}
|
||||
if profile != "" {
|
||||
continue
|
||||
}
|
||||
if prompt.DefaultProfileID == nil || *prompt.DefaultProfileID == "" {
|
||||
// A module may select a profile dynamically when constructing its
|
||||
// actual request. No declared selection is not a missing resource.
|
||||
continue
|
||||
}
|
||||
if _, err := engine.InspectProfile(ctx, *prompt.DefaultProfileID); err != nil {
|
||||
findings = append(findings, fmt.Errorf("%s prompt %q default profile %q: %w", scope, id, *prompt.DefaultProfileID, err))
|
||||
}
|
||||
}
|
||||
}
|
||||
input, _ := catalog.Inputs.Spec(resolved.Input.Module)
|
||||
check(pipeline.StageInput, "", "", resolved.Input.Module, "", resolved.InputExecutionClass, resolved.Input.LLMProfile, input.Prompts)
|
||||
check(pipeline.StageChunk, "", "", resolved.Chunk.Module, "", resolved.ChunkExecutionClass, resolved.Chunk.LLMProfile, resolved.ChunkPrompts)
|
||||
steps := make(map[string]string)
|
||||
for _, step := range resolved.Steps {
|
||||
for _, lane := range step.ArtifactLanes {
|
||||
steps[lane.ID] = step.ID
|
||||
check(pipeline.StageExtract, step.ID, lane.ID, lane.Extract.Module, "", lane.ExtractExecutionClass, lane.Extract.LLMProfile, lane.ExtractPrompts)
|
||||
check(pipeline.StageMerge, step.ID, lane.ID, lane.Merge.Module, "", lane.MergeExecutionClass, lane.Merge.LLMProfile, lane.MergePrompts)
|
||||
check(pipeline.StageNormalize, step.ID, lane.ID, lane.Normalize.Module, "", lane.NormalizeExecutionClass, lane.Normalize.LLMProfile, lane.NormalizePrompts)
|
||||
}
|
||||
}
|
||||
for _, chain := range resolved.ValidatorChains {
|
||||
for _, validator := range chain.Validators {
|
||||
check(chain.Stage, steps[chain.LaneID], chain.LaneID, chain.ModuleKey, validator.Binding.Module, validator.ExecutionClass, validator.Binding.LLMProfile, validator.Prompts)
|
||||
}
|
||||
}
|
||||
output, _ := catalog.Outputs.Spec(resolved.Output.Module)
|
||||
check(pipeline.StageOutput, "", "", resolved.Output.Module, "", resolved.OutputExecutionClass, resolved.Output.LLMProfile, output.Prompts)
|
||||
if err := ctx.Err(); err != nil {
|
||||
findings = append(findings, err)
|
||||
}
|
||||
return errors.Join(findings...)
|
||||
}
|
||||
163
internal/preflight/preflight_test.go
Normal file
163
internal/preflight/preflight_test.go
Normal file
@@ -0,0 +1,163 @@
|
||||
package preflight
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
type noGeneration struct{ t *testing.T }
|
||||
|
||||
func (client noGeneration) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
client.t.Fatal("offline preflight invoked a provider")
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func inspectionEngine(t *testing.T) *promptkit.Engine {
|
||||
t.Helper()
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{promptkit.SettingsInput(configkit.Bytes("offline resources", []byte(`profiles:
|
||||
- id: partial
|
||||
model: fixture
|
||||
api_key_env: NOTARIUS_PREFLIGHT_MUST_NOT_REQUIRE_THIS_CREDENTIAL
|
||||
api_key_required: true
|
||||
prompts:
|
||||
- id: initial
|
||||
default_profile: partial
|
||||
inputs: [{name: source, required: true}]
|
||||
messages: [{role: user, content: '{{input "source"}}'}]
|
||||
output: {format: text, validation_mode: none}
|
||||
- id: correction
|
||||
default_profile: partial
|
||||
messages: [{role: user, content: 'Correct the previous response.'}]
|
||||
output: {format: text, validation_mode: none}
|
||||
- id: validator
|
||||
default_profile: partial
|
||||
messages: [{role: user, content: 'Assess.'}]
|
||||
output: {format: text, validation_mode: none}
|
||||
- id: unknown-default
|
||||
default_profile: missing-profile
|
||||
messages: [{role: user, content: 'Select a profile.'}]
|
||||
output: {format: text, validation_mode: none}
|
||||
- id: no-default
|
||||
messages: [{role: user, content: 'Select a profile.'}]
|
||||
output: {format: text, validation_mode: none}
|
||||
`)))}}}}}, promptkit.WithBundledOpenRouter(false), promptkit.WithLLMClient(noGeneration{t}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return engine
|
||||
}
|
||||
|
||||
func resolvedFixture() pipeline.ResolvedPipeline {
|
||||
return pipeline.ResolvedPipeline{ID: "campaign", Steps: []pipeline.ResolvedPipelineStep{{ID: "identify", ArtifactLanes: []pipeline.ResolvedArtifactLane{{ID: "characters", Extract: pipeline.Binding("character-producer"), ExtractExecutionClass: contracts.ExecutionClassLLMBacked, ExtractPrompts: []string{"initial", "correction"}}}}},
|
||||
ValidatorChains: []pipeline.ResolvedValidatorChain{{Stage: pipeline.StageExtract, LaneID: "characters", ModuleKey: "character-producer", Validators: []pipeline.ResolvedValidator{{Binding: pipeline.Binding("domain-validator"), ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{"validator"}}}}},
|
||||
}
|
||||
}
|
||||
|
||||
func TestOfflinePreflightAcceptsPartialDefaultsWithoutExecutionInputs(t *testing.T) {
|
||||
engine := inspectionEngine(t)
|
||||
resolved := resolvedFixture()
|
||||
resolved.Steps[0].ArtifactLanes[0].ExtractReferences.Bindings = []pipeline.ReferenceBinding{{Source: "/not-present/reference.json"}}
|
||||
if !RequiresEngine(resolved) {
|
||||
t.Fatal("selected LLM roles were missed")
|
||||
}
|
||||
if err := Validate(context.Background(), resolved, pipeline.ModuleCatalog{}, engine); err != nil {
|
||||
t.Fatalf("partial profile or absent execution inputs rejected: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOfflinePreflightReportsEveryKnownResourceWithScopeAndCause(t *testing.T) {
|
||||
resolved := resolvedFixture()
|
||||
lane := &resolved.Steps[0].ArtifactLanes[0]
|
||||
lane.Extract.LLMProfile = "missing-explicit"
|
||||
lane.ExtractPrompts = []string{"missing-initial", "missing-correction"}
|
||||
resolved.ValidatorChains[0].Validators[0].Prompts = []string{"missing-validator"}
|
||||
err := Validate(context.Background(), resolved, pipeline.ModuleCatalog{}, inspectionEngine(t))
|
||||
if !errors.Is(err, promptkit.ErrPromptNotFound) || !errors.Is(err, promptkit.ErrProfileNotFound) {
|
||||
t.Fatalf("native causes lost: %v", err)
|
||||
}
|
||||
for _, value := range []string{"campaign", "identify", "characters", "extract", "character-producer", "domain-validator", "missing-explicit", "missing-initial", "missing-correction", "missing-validator"} {
|
||||
if !strings.Contains(err.Error(), value) {
|
||||
t.Fatalf("missing preflight context %q: %v", value, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestOfflinePreflightChecksApplicableProfilesButNotExecutionReadiness(t *testing.T) {
|
||||
engine := inspectionEngine(t)
|
||||
for _, test := range []struct {
|
||||
prompt, profile string
|
||||
want error
|
||||
}{
|
||||
{"unknown-default", "", promptkit.ErrProfileNotFound},
|
||||
{"no-default", "", nil},
|
||||
{"unknown-default", "partial", nil},
|
||||
{"no-default", "partial", nil},
|
||||
} {
|
||||
resolved := resolvedFixture()
|
||||
lane := &resolved.Steps[0].ArtifactLanes[0]
|
||||
lane.ExtractPrompts, lane.Extract.LLMProfile = []string{test.prompt}, test.profile
|
||||
err := Validate(context.Background(), resolved, pipeline.ModuleCatalog{}, engine)
|
||||
if test.want == nil && err != nil || test.want != nil && !errors.Is(err, test.want) {
|
||||
t.Fatalf("prompt=%q profile=%q: %v", test.prompt, test.profile, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestOfflinePreflightLeavesDynamicSelectionsForRuntime(t *testing.T) {
|
||||
resolved := resolvedFixture()
|
||||
resolved.Steps[0].ArtifactLanes[0].ExtractPrompts = nil
|
||||
resolved.ValidatorChains = nil
|
||||
if err := Validate(context.Background(), resolved, pipeline.ModuleCatalog{}, inspectionEngine(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOfflinePreflightIncludesInputOutputAndNestedValidatorsWithoutConstruction(t *testing.T) {
|
||||
catalog := pipeline.ModuleCatalog{Inputs: pipeline.NewInputAdapterRegistry(), Outputs: pipeline.NewOutputEncoderRegistry()}
|
||||
for _, stage := range []pipeline.ModuleStage{pipeline.StageInput, pipeline.StageOutput} {
|
||||
spec := pipeline.ModuleSpec{Key: string(stage), Stage: stage, ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{"missing-" + string(stage)}}
|
||||
var err error
|
||||
if stage == pipeline.StageInput {
|
||||
err = catalog.Inputs.RegisterWithSpec(spec, func() (contracts.InputAdapter, error) { t.Fatal("preflight constructed input"); return nil, nil })
|
||||
} else {
|
||||
err = catalog.Outputs.RegisterWithSpec(spec, func() (contracts.OutputEncoder, error) { t.Fatal("preflight constructed output"); return nil, nil })
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
resolved := pipeline.ResolvedPipeline{ID: "io", Input: pipeline.Binding("input"), InputExecutionClass: contracts.ExecutionClassLLMBacked, Output: pipeline.Binding("output"), OutputExecutionClass: contracts.ExecutionClassLLMBacked}
|
||||
err := Validate(context.Background(), resolved, catalog, inspectionEngine(t))
|
||||
if !errors.Is(err, promptkit.ErrPromptNotFound) || !strings.Contains(err.Error(), "missing-input") || !strings.Contains(err.Error(), "missing-output") {
|
||||
t.Fatalf("input/output declarations were missed: %v", err)
|
||||
}
|
||||
resolved = resolvedFixture()
|
||||
resolved.Steps[0].ArtifactLanes[0].ExtractExecutionClass = contracts.ExecutionClassDeterministic
|
||||
if !RequiresEngine(resolved) {
|
||||
t.Fatal("nested validator omitted from engine selection")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeterministicPreflightNeedsNoEngineAndCancellationStopsInspection(t *testing.T) {
|
||||
if RequiresEngine(pipeline.ResolvedPipeline{}) {
|
||||
t.Fatal("deterministic composition requires engine")
|
||||
}
|
||||
if err := Validate(context.Background(), pipeline.ResolvedPipeline{}, pipeline.ModuleCatalog{}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := Validate(context.Background(), resolvedFixture(), pipeline.ModuleCatalog{}, nil); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||
t.Fatalf("missing engine: %v", err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if err := Validate(ctx, resolvedFixture(), pipeline.ModuleCatalog{}, nil); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("cancellation lost: %v", err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user