164 lines
7.4 KiB
Go
164 lines
7.4 KiB
Go
|
|
package preflight
|
||
|
|
|
||
|
|
import (
|
||
|
|
"context"
|
||
|
|
"errors"
|
||
|
|
"strings"
|
||
|
|
"testing"
|
||
|
|
|
||
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||
|
|
"go.maximumdirect.net/configkit"
|
||
|
|
"go.maximumdirect.net/promptkit"
|
||
|
|
)
|
||
|
|
|
||
|
|
type noGeneration struct{ t *testing.T }
|
||
|
|
|
||
|
|
func (client noGeneration) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||
|
|
client.t.Fatal("offline preflight invoked a provider")
|
||
|
|
return nil, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func inspectionEngine(t *testing.T) *promptkit.Engine {
|
||
|
|
t.Helper()
|
||
|
|
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{promptkit.SettingsInput(configkit.Bytes("offline resources", []byte(`profiles:
|
||
|
|
- id: partial
|
||
|
|
model: fixture
|
||
|
|
api_key_env: NOTARIUS_PREFLIGHT_MUST_NOT_REQUIRE_THIS_CREDENTIAL
|
||
|
|
api_key_required: true
|
||
|
|
prompts:
|
||
|
|
- id: initial
|
||
|
|
default_profile: partial
|
||
|
|
inputs: [{name: source, required: true}]
|
||
|
|
messages: [{role: user, content: '{{input "source"}}'}]
|
||
|
|
output: {format: text, validation_mode: none}
|
||
|
|
- id: correction
|
||
|
|
default_profile: partial
|
||
|
|
messages: [{role: user, content: 'Correct the previous response.'}]
|
||
|
|
output: {format: text, validation_mode: none}
|
||
|
|
- id: validator
|
||
|
|
default_profile: partial
|
||
|
|
messages: [{role: user, content: 'Assess.'}]
|
||
|
|
output: {format: text, validation_mode: none}
|
||
|
|
- id: unknown-default
|
||
|
|
default_profile: missing-profile
|
||
|
|
messages: [{role: user, content: 'Select a profile.'}]
|
||
|
|
output: {format: text, validation_mode: none}
|
||
|
|
- id: no-default
|
||
|
|
messages: [{role: user, content: 'Select a profile.'}]
|
||
|
|
output: {format: text, validation_mode: none}
|
||
|
|
`)))}}}}}, promptkit.WithBundledOpenRouter(false), promptkit.WithLLMClient(noGeneration{t}))
|
||
|
|
if err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
return engine
|
||
|
|
}
|
||
|
|
|
||
|
|
func resolvedFixture() pipeline.ResolvedPipeline {
|
||
|
|
return pipeline.ResolvedPipeline{ID: "campaign", Steps: []pipeline.ResolvedPipelineStep{{ID: "identify", ArtifactLanes: []pipeline.ResolvedArtifactLane{{ID: "characters", Extract: pipeline.Binding("character-producer"), ExtractExecutionClass: contracts.ExecutionClassLLMBacked, ExtractPrompts: []string{"initial", "correction"}}}}},
|
||
|
|
ValidatorChains: []pipeline.ResolvedValidatorChain{{Stage: pipeline.StageExtract, LaneID: "characters", ModuleKey: "character-producer", Validators: []pipeline.ResolvedValidator{{Binding: pipeline.Binding("domain-validator"), ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{"validator"}}}}},
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestOfflinePreflightAcceptsPartialDefaultsWithoutExecutionInputs(t *testing.T) {
|
||
|
|
engine := inspectionEngine(t)
|
||
|
|
resolved := resolvedFixture()
|
||
|
|
resolved.Steps[0].ArtifactLanes[0].ExtractReferences.Bindings = []pipeline.ReferenceBinding{{Source: "/not-present/reference.json"}}
|
||
|
|
if !RequiresEngine(resolved) {
|
||
|
|
t.Fatal("selected LLM roles were missed")
|
||
|
|
}
|
||
|
|
if err := Validate(context.Background(), resolved, pipeline.ModuleCatalog{}, engine); err != nil {
|
||
|
|
t.Fatalf("partial profile or absent execution inputs rejected: %v", err)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestOfflinePreflightReportsEveryKnownResourceWithScopeAndCause(t *testing.T) {
|
||
|
|
resolved := resolvedFixture()
|
||
|
|
lane := &resolved.Steps[0].ArtifactLanes[0]
|
||
|
|
lane.Extract.LLMProfile = "missing-explicit"
|
||
|
|
lane.ExtractPrompts = []string{"missing-initial", "missing-correction"}
|
||
|
|
resolved.ValidatorChains[0].Validators[0].Prompts = []string{"missing-validator"}
|
||
|
|
err := Validate(context.Background(), resolved, pipeline.ModuleCatalog{}, inspectionEngine(t))
|
||
|
|
if !errors.Is(err, promptkit.ErrPromptNotFound) || !errors.Is(err, promptkit.ErrProfileNotFound) {
|
||
|
|
t.Fatalf("native causes lost: %v", err)
|
||
|
|
}
|
||
|
|
for _, value := range []string{"campaign", "identify", "characters", "extract", "character-producer", "domain-validator", "missing-explicit", "missing-initial", "missing-correction", "missing-validator"} {
|
||
|
|
if !strings.Contains(err.Error(), value) {
|
||
|
|
t.Fatalf("missing preflight context %q: %v", value, err)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestOfflinePreflightChecksApplicableProfilesButNotExecutionReadiness(t *testing.T) {
|
||
|
|
engine := inspectionEngine(t)
|
||
|
|
for _, test := range []struct {
|
||
|
|
prompt, profile string
|
||
|
|
want error
|
||
|
|
}{
|
||
|
|
{"unknown-default", "", promptkit.ErrProfileNotFound},
|
||
|
|
{"no-default", "", nil},
|
||
|
|
{"unknown-default", "partial", nil},
|
||
|
|
{"no-default", "partial", nil},
|
||
|
|
} {
|
||
|
|
resolved := resolvedFixture()
|
||
|
|
lane := &resolved.Steps[0].ArtifactLanes[0]
|
||
|
|
lane.ExtractPrompts, lane.Extract.LLMProfile = []string{test.prompt}, test.profile
|
||
|
|
err := Validate(context.Background(), resolved, pipeline.ModuleCatalog{}, engine)
|
||
|
|
if test.want == nil && err != nil || test.want != nil && !errors.Is(err, test.want) {
|
||
|
|
t.Fatalf("prompt=%q profile=%q: %v", test.prompt, test.profile, err)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestOfflinePreflightLeavesDynamicSelectionsForRuntime(t *testing.T) {
|
||
|
|
resolved := resolvedFixture()
|
||
|
|
resolved.Steps[0].ArtifactLanes[0].ExtractPrompts = nil
|
||
|
|
resolved.ValidatorChains = nil
|
||
|
|
if err := Validate(context.Background(), resolved, pipeline.ModuleCatalog{}, inspectionEngine(t)); err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestOfflinePreflightIncludesInputOutputAndNestedValidatorsWithoutConstruction(t *testing.T) {
|
||
|
|
catalog := pipeline.ModuleCatalog{Inputs: pipeline.NewInputAdapterRegistry(), Outputs: pipeline.NewOutputEncoderRegistry()}
|
||
|
|
for _, stage := range []pipeline.ModuleStage{pipeline.StageInput, pipeline.StageOutput} {
|
||
|
|
spec := pipeline.ModuleSpec{Key: string(stage), Stage: stage, ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{"missing-" + string(stage)}}
|
||
|
|
var err error
|
||
|
|
if stage == pipeline.StageInput {
|
||
|
|
err = catalog.Inputs.RegisterWithSpec(spec, func() (contracts.InputAdapter, error) { t.Fatal("preflight constructed input"); return nil, nil })
|
||
|
|
} else {
|
||
|
|
err = catalog.Outputs.RegisterWithSpec(spec, func() (contracts.OutputEncoder, error) { t.Fatal("preflight constructed output"); return nil, nil })
|
||
|
|
}
|
||
|
|
if err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
resolved := pipeline.ResolvedPipeline{ID: "io", Input: pipeline.Binding("input"), InputExecutionClass: contracts.ExecutionClassLLMBacked, Output: pipeline.Binding("output"), OutputExecutionClass: contracts.ExecutionClassLLMBacked}
|
||
|
|
err := Validate(context.Background(), resolved, catalog, inspectionEngine(t))
|
||
|
|
if !errors.Is(err, promptkit.ErrPromptNotFound) || !strings.Contains(err.Error(), "missing-input") || !strings.Contains(err.Error(), "missing-output") {
|
||
|
|
t.Fatalf("input/output declarations were missed: %v", err)
|
||
|
|
}
|
||
|
|
resolved = resolvedFixture()
|
||
|
|
resolved.Steps[0].ArtifactLanes[0].ExtractExecutionClass = contracts.ExecutionClassDeterministic
|
||
|
|
if !RequiresEngine(resolved) {
|
||
|
|
t.Fatal("nested validator omitted from engine selection")
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func TestDeterministicPreflightNeedsNoEngineAndCancellationStopsInspection(t *testing.T) {
|
||
|
|
if RequiresEngine(pipeline.ResolvedPipeline{}) {
|
||
|
|
t.Fatal("deterministic composition requires engine")
|
||
|
|
}
|
||
|
|
if err := Validate(context.Background(), pipeline.ResolvedPipeline{}, pipeline.ModuleCatalog{}, nil); err != nil {
|
||
|
|
t.Fatal(err)
|
||
|
|
}
|
||
|
|
if err := Validate(context.Background(), resolvedFixture(), pipeline.ModuleCatalog{}, nil); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||
|
|
t.Fatalf("missing engine: %v", err)
|
||
|
|
}
|
||
|
|
ctx, cancel := context.WithCancel(context.Background())
|
||
|
|
cancel()
|
||
|
|
if err := Validate(ctx, resolvedFixture(), pipeline.ModuleCatalog{}, nil); !errors.Is(err, context.Canceled) {
|
||
|
|
t.Fatalf("cancellation lost: %v", err)
|
||
|
|
}
|
||
|
|
}
|