package preflight import ( "context" "errors" "strings" "testing" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "go.maximumdirect.net/configkit" "go.maximumdirect.net/promptkit" ) type noGeneration struct{ t *testing.T } func (client noGeneration) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { client.t.Fatal("offline preflight invoked a provider") return nil, nil } func inspectionEngine(t *testing.T) *promptkit.Engine { t.Helper() engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{promptkit.SettingsInput(configkit.Bytes("offline resources", []byte(`profiles: - id: partial model: fixture api_key_env: NOTARIUS_PREFLIGHT_MUST_NOT_REQUIRE_THIS_CREDENTIAL api_key_required: true prompts: - id: initial default_profile: partial inputs: [{name: source, required: true}] messages: [{role: user, content: '{{input "source"}}'}] output: {format: text, validation_mode: none} - id: correction default_profile: partial messages: [{role: user, content: 'Correct the previous response.'}] output: {format: text, validation_mode: none} - id: validator default_profile: partial messages: [{role: user, content: 'Assess.'}] output: {format: text, validation_mode: none} - id: unknown-default default_profile: missing-profile messages: [{role: user, content: 'Select a profile.'}] output: {format: text, validation_mode: none} - id: no-default messages: [{role: user, content: 'Select a profile.'}] output: {format: text, validation_mode: none} `)))}}}}}, promptkit.WithBundledOpenRouter(false), promptkit.WithLLMClient(noGeneration{t})) if err != nil { t.Fatal(err) } return engine } func resolvedFixture() pipeline.ResolvedPipeline { return pipeline.ResolvedPipeline{ID: "campaign", Steps: []pipeline.ResolvedPipelineStep{{ID: "identify", ArtifactLanes: []pipeline.ResolvedArtifactLane{{ID: "characters", Extract: pipeline.Binding("character-producer"), ExtractExecutionClass: contracts.ExecutionClassLLMBacked, ExtractPrompts: []string{"initial", "correction"}}}}}, ValidatorChains: []pipeline.ResolvedValidatorChain{{Stage: pipeline.StageExtract, LaneID: "characters", ModuleKey: "character-producer", Validators: []pipeline.ResolvedValidator{{Binding: pipeline.Binding("domain-validator"), ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{"validator"}}}}}, } } func TestOfflinePreflightAcceptsPartialDefaultsWithoutExecutionInputs(t *testing.T) { engine := inspectionEngine(t) resolved := resolvedFixture() resolved.Steps[0].ArtifactLanes[0].ExtractReferences.Bindings = []pipeline.ReferenceBinding{{Source: "/not-present/reference.json"}} if !RequiresEngine(resolved) { t.Fatal("selected LLM roles were missed") } if err := Validate(context.Background(), resolved, pipeline.ModuleCatalog{}, engine); err != nil { t.Fatalf("partial profile or absent execution inputs rejected: %v", err) } } func TestOfflinePreflightReportsEveryKnownResourceWithScopeAndCause(t *testing.T) { resolved := resolvedFixture() lane := &resolved.Steps[0].ArtifactLanes[0] lane.Extract.LLMProfile = "missing-explicit" lane.ExtractPrompts = []string{"missing-initial", "missing-correction"} resolved.ValidatorChains[0].Validators[0].Prompts = []string{"missing-validator"} err := Validate(context.Background(), resolved, pipeline.ModuleCatalog{}, inspectionEngine(t)) if !errors.Is(err, promptkit.ErrPromptNotFound) || !errors.Is(err, promptkit.ErrProfileNotFound) { t.Fatalf("native causes lost: %v", err) } for _, value := range []string{"campaign", "identify", "characters", "extract", "character-producer", "domain-validator", "missing-explicit", "missing-initial", "missing-correction", "missing-validator"} { if !strings.Contains(err.Error(), value) { t.Fatalf("missing preflight context %q: %v", value, err) } } } func TestOfflinePreflightChecksApplicableProfilesButNotExecutionReadiness(t *testing.T) { engine := inspectionEngine(t) for _, test := range []struct { prompt, profile string want error }{ {"unknown-default", "", promptkit.ErrProfileNotFound}, {"no-default", "", nil}, {"unknown-default", "partial", nil}, {"no-default", "partial", nil}, } { resolved := resolvedFixture() lane := &resolved.Steps[0].ArtifactLanes[0] lane.ExtractPrompts, lane.Extract.LLMProfile = []string{test.prompt}, test.profile err := Validate(context.Background(), resolved, pipeline.ModuleCatalog{}, engine) if test.want == nil && err != nil || test.want != nil && !errors.Is(err, test.want) { t.Fatalf("prompt=%q profile=%q: %v", test.prompt, test.profile, err) } } } func TestOfflinePreflightLeavesDynamicSelectionsForRuntime(t *testing.T) { resolved := resolvedFixture() resolved.Steps[0].ArtifactLanes[0].ExtractPrompts = nil resolved.ValidatorChains = nil if err := Validate(context.Background(), resolved, pipeline.ModuleCatalog{}, inspectionEngine(t)); err != nil { t.Fatal(err) } } func TestOfflinePreflightIncludesInputOutputAndNestedValidatorsWithoutConstruction(t *testing.T) { catalog := pipeline.ModuleCatalog{Inputs: pipeline.NewInputAdapterRegistry(), Outputs: pipeline.NewOutputEncoderRegistry()} for _, stage := range []pipeline.ModuleStage{pipeline.StageInput, pipeline.StageOutput} { spec := pipeline.ModuleSpec{Key: string(stage), Stage: stage, ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{"missing-" + string(stage)}} var err error if stage == pipeline.StageInput { err = catalog.Inputs.RegisterWithSpec(spec, func() (contracts.InputAdapter, error) { t.Fatal("preflight constructed input"); return nil, nil }) } else { err = catalog.Outputs.RegisterWithSpec(spec, func() (contracts.OutputEncoder, error) { t.Fatal("preflight constructed output"); return nil, nil }) } if err != nil { t.Fatal(err) } } resolved := pipeline.ResolvedPipeline{ID: "io", Input: pipeline.Binding("input"), InputExecutionClass: contracts.ExecutionClassLLMBacked, Output: pipeline.Binding("output"), OutputExecutionClass: contracts.ExecutionClassLLMBacked} err := Validate(context.Background(), resolved, catalog, inspectionEngine(t)) if !errors.Is(err, promptkit.ErrPromptNotFound) || !strings.Contains(err.Error(), "missing-input") || !strings.Contains(err.Error(), "missing-output") { t.Fatalf("input/output declarations were missed: %v", err) } resolved = resolvedFixture() resolved.Steps[0].ArtifactLanes[0].ExtractExecutionClass = contracts.ExecutionClassDeterministic if !RequiresEngine(resolved) { t.Fatal("nested validator omitted from engine selection") } } func TestDeterministicPreflightNeedsNoEngineAndCancellationStopsInspection(t *testing.T) { if RequiresEngine(pipeline.ResolvedPipeline{}) { t.Fatal("deterministic composition requires engine") } if err := Validate(context.Background(), pipeline.ResolvedPipeline{}, pipeline.ModuleCatalog{}, nil); err != nil { t.Fatal(err) } if err := Validate(context.Background(), resolvedFixture(), pipeline.ModuleCatalog{}, nil); !errors.Is(err, promptkit.ErrInvalidConfig) { t.Fatalf("missing engine: %v", err) } ctx, cancel := context.WithCancel(context.Background()) cancel() if err := Validate(ctx, resolvedFixture(), pipeline.ModuleCatalog{}, nil); !errors.Is(err, context.Canceled) { t.Fatalf("cancellation lost: %v", err) } }