Files
notarius/internal/preflight/preflight_test.go

164 lines
7.4 KiB
Go
Raw Normal View History

package preflight
import (
"context"
"errors"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
type noGeneration struct{ t *testing.T }
func (client noGeneration) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
client.t.Fatal("offline preflight invoked a provider")
return nil, nil
}
func inspectionEngine(t *testing.T) *promptkit.Engine {
t.Helper()
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{promptkit.SettingsInput(configkit.Bytes("offline resources", []byte(`profiles:
- id: partial
model: fixture
api_key_env: NOTARIUS_PREFLIGHT_MUST_NOT_REQUIRE_THIS_CREDENTIAL
api_key_required: true
prompts:
- id: initial
default_profile: partial
inputs: [{name: source, required: true}]
messages: [{role: user, content: '{{input "source"}}'}]
output: {format: text, validation_mode: none}
- id: correction
default_profile: partial
messages: [{role: user, content: 'Correct the previous response.'}]
output: {format: text, validation_mode: none}
- id: validator
default_profile: partial
messages: [{role: user, content: 'Assess.'}]
output: {format: text, validation_mode: none}
- id: unknown-default
default_profile: missing-profile
messages: [{role: user, content: 'Select a profile.'}]
output: {format: text, validation_mode: none}
- id: no-default
messages: [{role: user, content: 'Select a profile.'}]
output: {format: text, validation_mode: none}
`)))}}}}}, promptkit.WithBundledOpenRouter(false), promptkit.WithLLMClient(noGeneration{t}))
if err != nil {
t.Fatal(err)
}
return engine
}
func resolvedFixture() pipeline.ResolvedPipeline {
return pipeline.ResolvedPipeline{ID: "campaign", Steps: []pipeline.ResolvedPipelineStep{{ID: "identify", ArtifactLanes: []pipeline.ResolvedArtifactLane{{ID: "characters", Extract: pipeline.Binding("character-producer"), ExtractExecutionClass: contracts.ExecutionClassLLMBacked, ExtractPrompts: []string{"initial", "correction"}}}}},
ValidatorChains: []pipeline.ResolvedValidatorChain{{Stage: pipeline.StageExtract, LaneID: "characters", ModuleKey: "character-producer", Validators: []pipeline.ResolvedValidator{{Binding: pipeline.Binding("domain-validator"), ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{"validator"}}}}},
}
}
func TestOfflinePreflightAcceptsPartialDefaultsWithoutExecutionInputs(t *testing.T) {
engine := inspectionEngine(t)
resolved := resolvedFixture()
resolved.Steps[0].ArtifactLanes[0].ExtractReferences.Bindings = []pipeline.ReferenceBinding{{Source: "/not-present/reference.json"}}
if !RequiresEngine(resolved) {
t.Fatal("selected LLM roles were missed")
}
if err := Validate(context.Background(), resolved, pipeline.ModuleCatalog{}, engine); err != nil {
t.Fatalf("partial profile or absent execution inputs rejected: %v", err)
}
}
func TestOfflinePreflightReportsEveryKnownResourceWithScopeAndCause(t *testing.T) {
resolved := resolvedFixture()
lane := &resolved.Steps[0].ArtifactLanes[0]
lane.Extract.LLMProfile = "missing-explicit"
lane.ExtractPrompts = []string{"missing-initial", "missing-correction"}
resolved.ValidatorChains[0].Validators[0].Prompts = []string{"missing-validator"}
err := Validate(context.Background(), resolved, pipeline.ModuleCatalog{}, inspectionEngine(t))
if !errors.Is(err, promptkit.ErrPromptNotFound) || !errors.Is(err, promptkit.ErrProfileNotFound) {
t.Fatalf("native causes lost: %v", err)
}
for _, value := range []string{"campaign", "identify", "characters", "extract", "character-producer", "domain-validator", "missing-explicit", "missing-initial", "missing-correction", "missing-validator"} {
if !strings.Contains(err.Error(), value) {
t.Fatalf("missing preflight context %q: %v", value, err)
}
}
}
func TestOfflinePreflightChecksApplicableProfilesButNotExecutionReadiness(t *testing.T) {
engine := inspectionEngine(t)
for _, test := range []struct {
prompt, profile string
want error
}{
{"unknown-default", "", promptkit.ErrProfileNotFound},
{"no-default", "", nil},
{"unknown-default", "partial", nil},
{"no-default", "partial", nil},
} {
resolved := resolvedFixture()
lane := &resolved.Steps[0].ArtifactLanes[0]
lane.ExtractPrompts, lane.Extract.LLMProfile = []string{test.prompt}, test.profile
err := Validate(context.Background(), resolved, pipeline.ModuleCatalog{}, engine)
if test.want == nil && err != nil || test.want != nil && !errors.Is(err, test.want) {
t.Fatalf("prompt=%q profile=%q: %v", test.prompt, test.profile, err)
}
}
}
func TestOfflinePreflightLeavesDynamicSelectionsForRuntime(t *testing.T) {
resolved := resolvedFixture()
resolved.Steps[0].ArtifactLanes[0].ExtractPrompts = nil
resolved.ValidatorChains = nil
if err := Validate(context.Background(), resolved, pipeline.ModuleCatalog{}, inspectionEngine(t)); err != nil {
t.Fatal(err)
}
}
func TestOfflinePreflightIncludesInputOutputAndNestedValidatorsWithoutConstruction(t *testing.T) {
catalog := pipeline.ModuleCatalog{Inputs: pipeline.NewInputAdapterRegistry(), Outputs: pipeline.NewOutputEncoderRegistry()}
for _, stage := range []pipeline.ModuleStage{pipeline.StageInput, pipeline.StageOutput} {
spec := pipeline.ModuleSpec{Key: string(stage), Stage: stage, ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{"missing-" + string(stage)}}
var err error
if stage == pipeline.StageInput {
err = catalog.Inputs.RegisterWithSpec(spec, func() (contracts.InputAdapter, error) { t.Fatal("preflight constructed input"); return nil, nil })
} else {
err = catalog.Outputs.RegisterWithSpec(spec, func() (contracts.OutputEncoder, error) { t.Fatal("preflight constructed output"); return nil, nil })
}
if err != nil {
t.Fatal(err)
}
}
resolved := pipeline.ResolvedPipeline{ID: "io", Input: pipeline.Binding("input"), InputExecutionClass: contracts.ExecutionClassLLMBacked, Output: pipeline.Binding("output"), OutputExecutionClass: contracts.ExecutionClassLLMBacked}
err := Validate(context.Background(), resolved, catalog, inspectionEngine(t))
if !errors.Is(err, promptkit.ErrPromptNotFound) || !strings.Contains(err.Error(), "missing-input") || !strings.Contains(err.Error(), "missing-output") {
t.Fatalf("input/output declarations were missed: %v", err)
}
resolved = resolvedFixture()
resolved.Steps[0].ArtifactLanes[0].ExtractExecutionClass = contracts.ExecutionClassDeterministic
if !RequiresEngine(resolved) {
t.Fatal("nested validator omitted from engine selection")
}
}
func TestDeterministicPreflightNeedsNoEngineAndCancellationStopsInspection(t *testing.T) {
if RequiresEngine(pipeline.ResolvedPipeline{}) {
t.Fatal("deterministic composition requires engine")
}
if err := Validate(context.Background(), pipeline.ResolvedPipeline{}, pipeline.ModuleCatalog{}, nil); err != nil {
t.Fatal(err)
}
if err := Validate(context.Background(), resolvedFixture(), pipeline.ModuleCatalog{}, nil); !errors.Is(err, promptkit.ErrInvalidConfig) {
t.Fatalf("missing engine: %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if err := Validate(ctx, resolvedFixture(), pipeline.ModuleCatalog{}, nil); !errors.Is(err, context.Canceled) {
t.Fatalf("cancellation lost: %v", err)
}
}