Add offline pipeline inspection and shared native resource preflight
This commit is contained in:
@@ -1503,6 +1503,27 @@ operations. Check meaningful evidence, not the complete native JSON shape.
|
||||
|
||||
## Stage 32: Add Offline Preflight And Pipeline Inspection Commands
|
||||
|
||||
**Status:** Complete. Added native pipeline list/show/validate declarations using
|
||||
the shared captured application load. List enumerates sorted configuration IDs
|
||||
without constructing a catalog or engine. Show uses ordinary domain resolution
|
||||
and reports steps, lanes, modules, controls, validator chains, prompt declarations,
|
||||
and reference bindings without expanding profiles or serializing runtime options.
|
||||
Validation resolves the same domain contracts and supplies the captured engine
|
||||
only for selected model work. The new preflight package inspects every known
|
||||
initial/correction/validator/input/output prompt and explicit/applicable default
|
||||
profile, preserves native causes, and aggregates pipeline/step/lane/validator
|
||||
context. Partial profiles pass; undeclared dynamic selections remain runtime
|
||||
checks. No request preparation, transcript/reference reads, credential lookup,
|
||||
provider call, module construction, or run-storage creation is involved.
|
||||
Focused real-kit preflight and CLI command tests, vet, repeated race checks,
|
||||
focused pipeline tests, production preflight/pipeline/config builds, and diff
|
||||
checks pass. Native configuration validation retains no pipeline-specific mode;
|
||||
added assembled help/config-command regressions. The full CLI suite, including
|
||||
those assembled regressions, remains blocked at setup by removed reporting-package
|
||||
imports and old ConfigKit's YAML plugin/limit dependency; it is not claimed passing.
|
||||
Scheduled cleanup and final verification must execute it. Native run-handler
|
||||
composition will reuse this preflight boundary in Stage 33.
|
||||
|
||||
**Goal:** execution and inspection share domain resolution and known dependencies.
|
||||
|
||||
**Inspect/change:** new `internal/preflight`, CLI pipeline declarations, catalog,
|
||||
|
||||
@@ -2,10 +2,9 @@ package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"go.maximumdirect.net/ckconfig"
|
||||
"go.maximumdirect.net/ckprompt"
|
||||
"go.maximumdirect.net/commandkit"
|
||||
@@ -42,6 +41,7 @@ func NewApplication(version string, options Options) (*commandkit.App, error) {
|
||||
definitions = ckprompt.WithNamespaces(definitions, promptkit.Namespaces{})
|
||||
declarations := []commandkit.Declaration{
|
||||
commandkit.Completion("notarius"),
|
||||
pipelineCommands(newState, func() (pipeline.ModuleCatalog, error) { return effectiveCatalog(options) }),
|
||||
ckconfig.Command(ckconfig.Suite[struct{}, *nativeLoaders]{
|
||||
NewState: newState,
|
||||
CheckInput: func(_ string, selection ckconfig.Selection, _ struct{}) error {
|
||||
@@ -53,10 +53,3 @@ func NewApplication(version string, options Options) (*commandkit.App, error) {
|
||||
declarations = append(declarations, ckprompt.Declarations(definitions)...)
|
||||
return commandkit.New(commandkit.Identity{Name: "notarius", Version: version}, declarations...)
|
||||
}
|
||||
|
||||
func checkConfigSelection(selection ckconfig.Selection) error {
|
||||
if selection.Config.Set && strings.TrimSpace(selection.Config.Value) == "" {
|
||||
return fmt.Errorf("--config must not be empty")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@ func TestNativeApplicationBuiltinsNeedNoConfiguration(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, args := range [][]string{{"--help"}, {"help"}, {"version"}, {"completion", "bash"}, {"completion", "zsh"}, {"completion", "fish"}, {"config", "--help"}} {
|
||||
for _, args := range [][]string{{"--help"}, {"help"}, {"version"}, {"completion", "bash"}, {"completion", "zsh"}, {"completion", "fish"}, {"config", "--help"}, {"pipeline", "--help"}, {"pipeline", "validate", "--help"}} {
|
||||
var out, diagnostics bytes.Buffer
|
||||
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &out, Err: &diagnostics})
|
||||
if outcome.Status != 0 || out.Len() == 0 {
|
||||
@@ -52,6 +52,7 @@ func TestNativeConfigurationScopesAndDiffUseSelectedLoadEvidence(t *testing.T) {
|
||||
{"differences", []string{"config", "diff", "--config", left, "--against-config", right, "--check"}, 3, "workers"},
|
||||
{"identical", []string{"config", "diff", "--config", left, "--against-config", left, "--check"}, 0, ""},
|
||||
{"empty", []string{"config", "show", "--config", ""}, 2, ""},
|
||||
{"configuration has no pipeline mode", []string{"config", "validate", "--config", left, "--pipeline", "demo"}, 2, ""},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
var out, diagnostics bytes.Buffer
|
||||
|
||||
@@ -31,6 +31,13 @@ type loadedSelection struct {
|
||||
engine *promptkit.Engine
|
||||
}
|
||||
|
||||
func checkConfigSelection(selection ckconfig.Selection) error {
|
||||
if selection.Config.Set && strings.TrimSpace(selection.Config.Value) == "" {
|
||||
return fmt.Errorf("--config must not be empty")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (loaders *nativeLoaders) application(ctx context.Context, selection ckconfig.Selection) (*configkit.Result[config.Document], error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
|
||||
143
internal/cli/pipeline_commands.go
Normal file
143
internal/cli/pipeline_commands.go
Normal file
@@ -0,0 +1,143 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"text/template"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/preflight"
|
||||
"go.maximumdirect.net/ckconfig"
|
||||
"go.maximumdirect.net/commandkit"
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
type pipelineInspectionInput struct {
|
||||
Selection ckconfig.Selection
|
||||
PipelineID string
|
||||
}
|
||||
|
||||
func configurationSelectionFlags[I any](destination func(*I) *ckconfig.Selection) []commandkit.Flag[I] {
|
||||
return []commandkit.Flag[I]{
|
||||
commandkit.ValueFlag(commandkit.FlagMeta{Name: "config", Usage: "Select the application configuration.", Completion: commandkit.CompleteFiles()}, (*string)(nil), func(value string) (string, error) {
|
||||
if strings.TrimSpace(value) == "" {
|
||||
return "", fmt.Errorf("--config must not be empty")
|
||||
}
|
||||
return value, nil
|
||||
}, func(input *I, value commandkit.Supplied[string]) { destination(input).Config = value }),
|
||||
commandkit.ValueFlag(commandkit.FlagMeta{Name: "overlay", Usage: "Select a configuration overlay.", Completion: commandkit.CompleteFiles()}, (*string)(nil), func(value string) (string, error) { return value, nil }, func(input *I, value commandkit.Supplied[string]) { destination(input).Overlay = value }),
|
||||
commandkit.RepeatFlag(commandkit.FlagMeta{Name: "set", Usage: "Override POINTER=JSON."}, []configkit.Override(nil), parseConfigurationOverride, func(input *I, value commandkit.Supplied[[]configkit.Override]) {
|
||||
destination(input).Overrides = value.Value
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
func parseConfigurationOverride(raw string) (configkit.Override, error) {
|
||||
pointer, encoded, ok := strings.Cut(raw, "=")
|
||||
if !ok {
|
||||
return configkit.Override{}, fmt.Errorf("override requires POINTER=JSON")
|
||||
}
|
||||
path, err := configkit.ParsePath(pointer)
|
||||
if err != nil {
|
||||
return configkit.Override{}, err
|
||||
}
|
||||
value, err := commandkit.ParseCLIJSON(encoded)
|
||||
if err != nil {
|
||||
return configkit.Override{}, err
|
||||
}
|
||||
return configkit.Override{Path: path, Value: value, Flag: "--set"}, nil
|
||||
}
|
||||
|
||||
func pipelineCommands(newState func() *nativeLoaders, catalog func() (pipeline.ModuleCatalog, error)) commandkit.Command {
|
||||
command := func(operation, summary string) commandkit.Command {
|
||||
var arguments []commandkit.Argument[pipelineInspectionInput]
|
||||
if operation != "list" {
|
||||
arguments = []commandkit.Argument[pipelineInspectionInput]{commandkit.ValueArgument(commandkit.ArgumentMeta{Name: "pipeline-id", Usage: "Configured pipeline identifier."}, true, func(value string) (string, error) {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
return "", fmt.Errorf("pipeline id must not be empty")
|
||||
}
|
||||
return value, nil
|
||||
}, func(input *pipelineInspectionInput, value commandkit.Supplied[string]) {
|
||||
input.PipelineID = value.Value
|
||||
})}
|
||||
}
|
||||
return commandkit.NewCommand(commandkit.CommandSpec[pipelineInspectionInput]{Name: operation, Summary: summary,
|
||||
Flags: configurationSelectionFlags(func(input *pipelineInspectionInput) *ckconfig.Selection { return &input.Selection }), Arguments: arguments,
|
||||
CheckInput: func(input pipelineInspectionInput) error { return checkConfigSelection(input.Selection) },
|
||||
Run: func(invocation commandkit.Invocation, input pipelineInspectionInput) commandkit.Outcome {
|
||||
state := newState()
|
||||
result, err := state.application(invocation.Context, input.Selection)
|
||||
if err != nil {
|
||||
return pipelineConfigurationFailure(err)
|
||||
}
|
||||
if operation == "list" {
|
||||
ids := make([]string, 0, len(result.Config.Pipelines))
|
||||
for id := range result.Config.Pipelines {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
sort.Strings(ids)
|
||||
return reportPipelineInspection(invocation, commandkit.Report{Value: ids})
|
||||
}
|
||||
loaded, err := config.FromResult(result)
|
||||
if err != nil {
|
||||
return pipelineInvalidConfiguration(err)
|
||||
}
|
||||
modules, err := catalog()
|
||||
if err != nil {
|
||||
return commandkit.Failure(err.Error()).WithCause(err)
|
||||
}
|
||||
effective, err := loaded.RuntimeConfig().Resolve(config.ResolveInput{PipelineID: input.PipelineID, Catalog: modules})
|
||||
if err != nil {
|
||||
return pipelineInvalidConfiguration(err)
|
||||
}
|
||||
if operation == "show" {
|
||||
return reportPipelineInspection(invocation, commandkit.Report{Value: describePipeline(effective.ResolvedPipeline, modules)})
|
||||
}
|
||||
var engine *promptkit.Engine
|
||||
if preflight.RequiresEngine(effective.ResolvedPipeline) {
|
||||
engine, err = state.engine(invocation.Context, input.Selection)
|
||||
if err != nil {
|
||||
return pipelineConfigurationFailure(fmt.Errorf("pipeline %q native resources: %w", input.PipelineID, err))
|
||||
}
|
||||
}
|
||||
if err := preflight.Validate(invocation.Context, effective.ResolvedPipeline, modules, engine); err != nil {
|
||||
return pipelineInvalidConfiguration(err)
|
||||
}
|
||||
return reportPipelineInspection(invocation, commandkit.Report{Value: struct {
|
||||
Pipeline string `json:"pipeline"`
|
||||
Valid bool `json:"valid"`
|
||||
}{input.PipelineID, true}, TextTemplate: pipelineValidationTemplate})
|
||||
},
|
||||
})
|
||||
}
|
||||
return commandkit.Group("pipeline", "Inspect, validate, or execute configured pipelines.", command("list", "List configured pipeline identifiers."), command("show", "Show a resolved pipeline composition."), command("validate", "Validate known pipeline dependencies offline."))
|
||||
}
|
||||
|
||||
var pipelineValidationTemplate = template.Must(commandkit.ParseTextTemplate("pipeline-validation", "Pipeline {{printf \"%q\" .Pipeline}} is valid.\n", nil))
|
||||
|
||||
func reportPipelineInspection(invocation commandkit.Invocation, report commandkit.Report) commandkit.Outcome {
|
||||
if err := invocation.Reporter().Report(report); err != nil {
|
||||
return commandkit.Failure("Could not report pipeline inspection.").WithCause(err)
|
||||
}
|
||||
return commandkit.Success()
|
||||
}
|
||||
|
||||
func pipelineConfigurationFailure(err error) commandkit.Outcome {
|
||||
if ckconfig.ClassifyConfigurationError(err) == commandkit.OutcomeUsage {
|
||||
return commandkit.Usage(err.Error()).WithCause(err)
|
||||
}
|
||||
return commandkit.Failure(err.Error()).WithCause(err)
|
||||
}
|
||||
|
||||
func pipelineInvalidConfiguration(err error) commandkit.Outcome {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return commandkit.Failure(err.Error()).WithCause(err)
|
||||
}
|
||||
return commandkit.Usage(err.Error()).WithCause(err)
|
||||
}
|
||||
276
internal/cli/pipeline_commands_test.go
Normal file
276
internal/cli/pipeline_commands_test.go
Normal file
@@ -0,0 +1,276 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"go.maximumdirect.net/commandkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
const inspectionConfiguration = `version: 6
|
||||
output: {directory: output-must-not-exist}
|
||||
cache_directory: cache-must-not-exist
|
||||
debug: {directory: debug-must-not-exist}
|
||||
pipelines:
|
||||
demo:
|
||||
input: {module: fixture/input}
|
||||
chunk: {module: fixture/chunk}
|
||||
output: {module: fixture/output}
|
||||
steps:
|
||||
- id: build
|
||||
artifacts:
|
||||
notes:
|
||||
extract:
|
||||
module: fixture/extract
|
||||
references: {context: {path: reference-must-not-exist.txt}}
|
||||
validators: [{module: fixture/validator}]
|
||||
merge: {module: fixture/merge}
|
||||
normalize: {module: fixture/normalize}
|
||||
promptkit:
|
||||
profiles:
|
||||
- id: partial
|
||||
model: fixture-model
|
||||
api_key_env: NOTARIUS_INSPECTION_NO_CREDENTIAL_REQUIRED
|
||||
api_key_required: true
|
||||
prompts:
|
||||
- id: inspect-initial
|
||||
default_profile: partial
|
||||
inputs: [{name: source, required: true}]
|
||||
messages: [{role: user, content: '{{input "source"}}'}]
|
||||
output: {format: text, validation_mode: none}
|
||||
- id: inspect-correction
|
||||
default_profile: partial
|
||||
messages: [{role: user, content: Correct.}]
|
||||
output: {format: text, validation_mode: none}
|
||||
- id: inspect-validator
|
||||
default_profile: partial
|
||||
messages: [{role: user, content: Assess.}]
|
||||
output: {format: text, validation_mode: none}
|
||||
`
|
||||
|
||||
type inspectionArtifact struct {
|
||||
Value string `json:"value"`
|
||||
}
|
||||
type inspectionCodec struct{}
|
||||
|
||||
func (inspectionCodec) Kind() contracts.ArtifactKind { return "test/inspection" }
|
||||
func (inspectionCodec) MediaType() string { return "application/json" }
|
||||
func (inspectionCodec) Schema() contracts.ArtifactSchema {
|
||||
return contracts.ArtifactSchema{ID: "test.inspection", Name: "inspection", Version: "v1", JSONSchema: []byte(`{"type":"object","properties":{"value":{"type":"string"}},"required":["value"]}`)}
|
||||
}
|
||||
func (inspectionCodec) Encode(value inspectionArtifact) ([]byte, error) { return json.Marshal(value) }
|
||||
func (codec inspectionCodec) EncodeCandidate(value inspectionArtifact) ([]byte, error) {
|
||||
return codec.Encode(value)
|
||||
}
|
||||
func (codec inspectionCodec) DecodeCandidate(content []byte) (inspectionArtifact, error) {
|
||||
return codec.Decode(content)
|
||||
}
|
||||
func (inspectionCodec) Decode(content []byte) (inspectionArtifact, error) {
|
||||
var value inspectionArtifact
|
||||
err := json.Unmarshal(content, &value)
|
||||
return value, err
|
||||
}
|
||||
|
||||
func inspectionCatalog(t *testing.T, class contracts.ExecutionClass) pipeline.ModuleCatalog {
|
||||
t.Helper()
|
||||
catalog := pipeline.ModuleCatalog{Inputs: pipeline.NewInputAdapterRegistry(), Chunkers: pipeline.NewChunkerRegistry(), ArtifactCodecs: pipeline.NewArtifactCodecRegistry(), Extractors: pipeline.NewExtractorRegistry(), Mergers: pipeline.NewMergerRegistry(), Normalizers: pipeline.NewNormalizerRegistry(), Validators: pipeline.NewValidatorRegistry(), ValidatorChains: pipeline.NewValidatorChainRegistry(), Outputs: pipeline.NewOutputEncoderRegistry()}
|
||||
check := func(err error) {
|
||||
t.Helper()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
spec := func(stage pipeline.ModuleStage, key string) pipeline.ModuleSpec {
|
||||
return pipeline.ModuleSpec{Key: key, Stage: stage, ExecutionClass: contracts.ExecutionClassDeterministic}
|
||||
}
|
||||
check(catalog.Inputs.RegisterWithSpec(spec(pipeline.StageInput, "fixture/input"), func() (contracts.InputAdapter, error) { t.Fatal("inspection constructed input"); return nil, nil }))
|
||||
check(catalog.Chunkers.RegisterWithSpec(spec(pipeline.StageChunk, "fixture/chunk"), func() (contracts.Chunker, error) { t.Fatal("inspection constructed chunker"); return nil, nil }))
|
||||
check(catalog.Outputs.RegisterWithSpec(spec(pipeline.StageOutput, "fixture/output"), func() (contracts.OutputEncoder, error) { t.Fatal("inspection constructed output"); return nil, nil }))
|
||||
check(pipeline.RegisterArtifactCodec(catalog.ArtifactCodecs, inspectionCodec{}))
|
||||
extract := spec(pipeline.StageExtract, "fixture/extract")
|
||||
extract.ArtifactKind, extract.ExecutionClass, extract.Prompts = "test/inspection", class, []string{"inspect-initial", "inspect-correction"}
|
||||
extract.ReferenceSlots = []contracts.ReferenceSlot{{Name: "context", Required: true, AcceptedMediaTypes: []string{"text/plain"}}}
|
||||
check(pipeline.RegisterExtractor(catalog.Extractors, extract, func() (contracts.Extractor[inspectionArtifact], error) {
|
||||
t.Fatal("inspection constructed extractor")
|
||||
return nil, nil
|
||||
}))
|
||||
merge := spec(pipeline.StageMerge, "fixture/merge")
|
||||
merge.ArtifactKind = "test/inspection"
|
||||
check(pipeline.RegisterMerger(catalog.Mergers, merge, func() (contracts.Merger[inspectionArtifact], error) {
|
||||
t.Fatal("inspection constructed merger")
|
||||
return nil, nil
|
||||
}))
|
||||
normalize := spec(pipeline.StageNormalize, "fixture/normalize")
|
||||
normalize.ArtifactKind = "test/inspection"
|
||||
check(pipeline.RegisterNormalizer(catalog.Normalizers, normalize, func() (contracts.Normalizer[inspectionArtifact], error) {
|
||||
t.Fatal("inspection constructed normalizer")
|
||||
return nil, nil
|
||||
}))
|
||||
check(pipeline.RegisterTypedValidator(catalog.Validators, "test/inspection", pipeline.ValidatorSpec{Key: "fixture/validator", ExecutionClass: class, Prompts: []string{"inspect-validator"}}, func() (contracts.TypedValidator[inspectionArtifact], error) {
|
||||
t.Fatal("inspection constructed validator")
|
||||
return nil, nil
|
||||
}))
|
||||
return catalog
|
||||
}
|
||||
|
||||
type inspectionProvider struct{ t *testing.T }
|
||||
|
||||
func (provider inspectionProvider) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
provider.t.Fatal("inspection called provider")
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func writeInspectionConfiguration(t *testing.T, content string) string {
|
||||
t.Helper()
|
||||
file := filepath.Join(t.TempDir(), "config.yml")
|
||||
if err := os.WriteFile(file, []byte(content), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return file
|
||||
}
|
||||
|
||||
func newInspectionApplication(t *testing.T, catalog func() (pipeline.ModuleCatalog, error)) (*commandkit.App, **nativeLoaders) {
|
||||
t.Helper()
|
||||
var last *nativeLoaders
|
||||
app, err := commandkit.New(commandkit.Identity{Name: "notarius"}, pipelineCommands(func() *nativeLoaders {
|
||||
last = &nativeLoaders{engineOptions: []promptkit.Option{promptkit.WithLLMClient(inspectionProvider{t})}}
|
||||
return last
|
||||
}, catalog))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return app, &last
|
||||
}
|
||||
|
||||
func TestPipelineListOnlyEnumeratesCapturedConfiguration(t *testing.T) {
|
||||
content := strings.Replace(inspectionConfiguration, " demo:", " zeta:", 1)
|
||||
content = strings.Replace(content, "promptkit:", " alpha:\n input: {module: unavailable}\n artifacts:\n value:\n extract: {module: unavailable}\npromptkit:", 1)
|
||||
file := writeInspectionConfiguration(t, content)
|
||||
app, state := newInspectionApplication(t, func() (pipeline.ModuleCatalog, error) {
|
||||
t.Fatal("list constructed a catalog")
|
||||
return pipeline.ModuleCatalog{}, nil
|
||||
})
|
||||
var out, diagnostics bytes.Buffer
|
||||
outcome := app.Execute(context.Background(), []string{"pipeline", "list", "--config", file, "--format", "json"}, commandkit.Streams{Out: &out, Err: &diagnostics})
|
||||
var ids []string
|
||||
if outcome.Status != 0 || json.Unmarshal(out.Bytes(), &ids) != nil || !reflect.DeepEqual(ids, []string{"alpha", "zeta"}) {
|
||||
t.Fatalf("list=%#v %s %s", outcome, out.String(), diagnostics.String())
|
||||
}
|
||||
if len((*state).selections) != 1 || (*state).selections[0].engine != nil {
|
||||
t.Fatal("list expanded native resources")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPipelineShowResolvesCompositionWithoutEngineOrExecutionInputs(t *testing.T) {
|
||||
file := writeInspectionConfiguration(t, inspectionConfiguration)
|
||||
catalog := inspectionCatalog(t, contracts.ExecutionClassLLMBacked)
|
||||
app, state := newInspectionApplication(t, func() (pipeline.ModuleCatalog, error) { return catalog, nil })
|
||||
var out, diagnostics bytes.Buffer
|
||||
outcome := app.Execute(context.Background(), []string{"pipeline", "show", "demo", "--config", file, "--format", "json"}, commandkit.Streams{Out: &out, Err: &diagnostics})
|
||||
var report pipelineDescription
|
||||
if err := json.Unmarshal(out.Bytes(), &report); err != nil || outcome.Status != 0 {
|
||||
t.Fatalf("show=%#v %s %s %v", outcome, out.String(), diagnostics.String(), err)
|
||||
}
|
||||
if report.Pipeline != "demo" || len(report.Steps) != 1 || report.Steps[0].ID != "build" || len(report.Steps[0].Lanes) != 1 || report.Steps[0].Lanes[0].Extract.Module != "fixture/extract" || len(report.Steps[0].Lanes[0].Extract.References) != 1 || len(report.Validators) == 0 {
|
||||
t.Fatalf("missing resolved composition: %#v", report)
|
||||
}
|
||||
if (*state).selections[0].engine != nil {
|
||||
t.Fatal("show expanded profiles by constructing engine")
|
||||
}
|
||||
assertInspectionNoRunStorage(t, file)
|
||||
}
|
||||
|
||||
func TestPipelineValidateChecksNativeDeclarationsOfflineAndUsesOneLoad(t *testing.T) {
|
||||
for _, class := range []contracts.ExecutionClass{contracts.ExecutionClassDeterministic, contracts.ExecutionClassLLMBacked} {
|
||||
file := writeInspectionConfiguration(t, inspectionConfiguration)
|
||||
catalog := inspectionCatalog(t, class)
|
||||
app, state := newInspectionApplication(t, func() (pipeline.ModuleCatalog, error) { return catalog, nil })
|
||||
var out, diagnostics bytes.Buffer
|
||||
outcome := app.Execute(context.Background(), []string{"pipeline", "validate", "demo", "--config", file}, commandkit.Streams{Out: &out, Err: &diagnostics})
|
||||
if outcome.Status != 0 || !strings.Contains(out.String(), "demo") {
|
||||
t.Fatalf("validate=%#v %s %s", outcome, out.String(), diagnostics.String())
|
||||
}
|
||||
if len((*state).selections) != 1 || ((*state).selections[0].engine != nil) != (class == contracts.ExecutionClassLLMBacked) {
|
||||
t.Fatal("validation loaded twice or built an unnecessary engine")
|
||||
}
|
||||
assertInspectionNoRunStorage(t, file)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPipelineValidateRejectsKnownMissingResourcesWithScope(t *testing.T) {
|
||||
for _, id := range []string{"inspect-initial", "inspect-correction", "inspect-validator"} {
|
||||
file := writeInspectionConfiguration(t, strings.Replace(inspectionConfiguration, "id: "+id, "id: unused-"+id, 1))
|
||||
catalog := inspectionCatalog(t, contracts.ExecutionClassLLMBacked)
|
||||
app, _ := newInspectionApplication(t, func() (pipeline.ModuleCatalog, error) { return catalog, nil })
|
||||
var out, diagnostics bytes.Buffer
|
||||
outcome := app.Execute(context.Background(), []string{"pipeline", "validate", "demo", "--config", file}, commandkit.Streams{Out: &out, Err: &diagnostics})
|
||||
if outcome.Status != 2 || !errors.Is(outcome.Cause, promptkit.ErrPromptNotFound) || !strings.Contains(diagnostics.String(), id) || !strings.Contains(diagnostics.String(), "build") || !strings.Contains(diagnostics.String(), "notes") {
|
||||
t.Fatalf("known dependency=%#v %s", outcome, diagnostics.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPipelineInspectionUsesNativeArgumentsSelectionsAndDelivery(t *testing.T) {
|
||||
file := writeInspectionConfiguration(t, inspectionConfiguration)
|
||||
catalog := inspectionCatalog(t, contracts.ExecutionClassLLMBacked)
|
||||
app, _ := newInspectionApplication(t, func() (pipeline.ModuleCatalog, error) { return catalog, nil })
|
||||
for _, test := range []struct {
|
||||
args []string
|
||||
status int
|
||||
}{
|
||||
{[]string{"pipeline", "show"}, 2},
|
||||
{[]string{"pipeline", "show", "demo", "--config", "", "--config", file}, 2},
|
||||
{[]string{"pipeline", "show", "unknown", "--config", file}, 2},
|
||||
{[]string{"pipeline", "show", "demo", "--config", file, "--set", `/pipelines/demo/steps/0/artifacts/notes/extract/options={"unsupported":true}`}, 2},
|
||||
{[]string{"pipeline", "validate", "demo", "--config", file, "--set", `/pipelines/demo/llm_profile="missing-profile"`}, 2},
|
||||
{[]string{"pipeline", "show", "demo", "--config", file, "--set", `/pipelines/demo/steps/0/artifacts/notes/extract/references=null`}, 2},
|
||||
{[]string{"pipeline", "list", "--config", file, "--set", `/pipelines/demo/input/module="another-input"`}, 0},
|
||||
{[]string{"pipeline", "validate", "demo", "--config", file, "--quiet"}, 0},
|
||||
{[]string{"pipeline", "--help"}, 0},
|
||||
} {
|
||||
var out, diagnostics bytes.Buffer
|
||||
outcome := app.Execute(context.Background(), test.args, commandkit.Streams{Out: &out, Err: &diagnostics})
|
||||
if outcome.Status != test.status {
|
||||
t.Fatalf("%v: %#v %s", test.args, outcome, diagnostics.String())
|
||||
}
|
||||
if test.args[len(test.args)-1] == "--quiet" && out.Len() != 0 {
|
||||
t.Fatal("quiet pipeline inspection wrote stdout")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type inspectionFailWriter struct{ err error }
|
||||
|
||||
func (writer inspectionFailWriter) Write([]byte) (int, error) { return 0, writer.err }
|
||||
|
||||
func TestPipelineInspectionReportFailureRemainsOperational(t *testing.T) {
|
||||
file := writeInspectionConfiguration(t, inspectionConfiguration)
|
||||
app, _ := newInspectionApplication(t, func() (pipeline.ModuleCatalog, error) {
|
||||
t.Fatal("list loaded catalog")
|
||||
return pipeline.ModuleCatalog{}, nil
|
||||
})
|
||||
primary := errors.New("inspection stdout delivery failed")
|
||||
var diagnostics bytes.Buffer
|
||||
outcome := app.Execute(context.Background(), []string{"pipeline", "list", "--config", file}, commandkit.Streams{Out: inspectionFailWriter{primary}, Err: &diagnostics})
|
||||
if outcome.Status != 1 || !errors.Is(outcome.Cause, primary) {
|
||||
t.Fatalf("delivery failure classified as invalid configuration: %#v %s", outcome, diagnostics.String())
|
||||
}
|
||||
}
|
||||
|
||||
func assertInspectionNoRunStorage(t *testing.T, file string) {
|
||||
t.Helper()
|
||||
entries, err := os.ReadDir(filepath.Dir(file))
|
||||
if err != nil || len(entries) != 1 || entries[0].Name() != "config.yml" {
|
||||
t.Fatalf("inspection read/materialized inputs or created storage: %v %v", entries, err)
|
||||
}
|
||||
}
|
||||
72
internal/cli/pipeline_description.go
Normal file
72
internal/cli/pipeline_description.go
Normal file
@@ -0,0 +1,72 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
)
|
||||
|
||||
// Inspection reports identities and resolved composition, not another effective
|
||||
// configuration or profile expansion. ConfigKit owns secret-aware settings views.
|
||||
type pipelineModuleDescription struct {
|
||||
Module string `json:"module"`
|
||||
ExecutionClass contracts.ExecutionClass `json:"execution_class"`
|
||||
LLMProfile string `json:"llm_profile,omitempty"`
|
||||
Controls *contracts.ProducerControls `json:"controls,omitempty"`
|
||||
Prompts []string `json:"prompts,omitempty"`
|
||||
References []pipeline.ReferenceBinding `json:"references,omitempty"`
|
||||
}
|
||||
|
||||
type pipelineLaneDescription struct {
|
||||
ID string `json:"id"`
|
||||
ArtifactKind contracts.ArtifactKind `json:"artifact_kind"`
|
||||
Extract pipelineModuleDescription `json:"extract"`
|
||||
Merge pipelineModuleDescription `json:"merge"`
|
||||
Normalize pipelineModuleDescription `json:"normalize"`
|
||||
}
|
||||
|
||||
type pipelineStepDescription struct {
|
||||
ID string `json:"id"`
|
||||
Lanes []pipelineLaneDescription `json:"lanes"`
|
||||
}
|
||||
type pipelineValidatorDescription struct {
|
||||
Stage pipeline.ModuleStage `json:"stage"`
|
||||
Lane string `json:"lane,omitempty"`
|
||||
Module string `json:"module"`
|
||||
Validators []pipelineModuleDescription `json:"validators"`
|
||||
}
|
||||
type pipelineDescription struct {
|
||||
Pipeline string `json:"pipeline"`
|
||||
Input pipelineModuleDescription `json:"input"`
|
||||
Chunk pipelineModuleDescription `json:"chunk"`
|
||||
Steps []pipelineStepDescription `json:"steps"`
|
||||
Validators []pipelineValidatorDescription `json:"validators,omitempty"`
|
||||
Output pipelineModuleDescription `json:"output"`
|
||||
}
|
||||
|
||||
func describePipeline(resolved pipeline.ResolvedPipeline, catalog pipeline.ModuleCatalog) pipelineDescription {
|
||||
module := func(binding pipeline.ModuleBinding, class contracts.ExecutionClass, prompts []string, controls *contracts.ProducerControls, references []pipeline.ReferenceBinding) pipelineModuleDescription {
|
||||
return pipelineModuleDescription{Module: binding.Module, ExecutionClass: class, LLMProfile: binding.LLMProfile, Prompts: prompts, Controls: controls, References: references}
|
||||
}
|
||||
input, _ := catalog.Inputs.Spec(resolved.Input.Module)
|
||||
output, _ := catalog.Outputs.Spec(resolved.Output.Module)
|
||||
report := pipelineDescription{Pipeline: resolved.ID, Input: module(resolved.Input, resolved.InputExecutionClass, input.Prompts, nil, nil),
|
||||
Chunk: module(resolved.Chunk, resolved.ChunkExecutionClass, resolved.ChunkPrompts, &resolved.ChunkControls, resolved.ChunkReferences.Bindings), Output: module(resolved.Output, resolved.OutputExecutionClass, output.Prompts, nil, nil)}
|
||||
for _, step := range resolved.Steps {
|
||||
description := pipelineStepDescription{ID: step.ID, Lanes: make([]pipelineLaneDescription, 0, len(step.ArtifactLanes))}
|
||||
for _, lane := range step.ArtifactLanes {
|
||||
description.Lanes = append(description.Lanes, pipelineLaneDescription{ID: lane.ID, ArtifactKind: lane.ArtifactKind,
|
||||
Extract: module(lane.Extract, lane.ExtractExecutionClass, lane.ExtractPrompts, &lane.ExtractControls, lane.ExtractReferences.Bindings),
|
||||
Merge: module(lane.Merge, lane.MergeExecutionClass, lane.MergePrompts, &lane.MergeControls, lane.MergeReferences.Bindings),
|
||||
Normalize: module(lane.Normalize, lane.NormalizeExecutionClass, lane.NormalizePrompts, &lane.NormalizeControls, lane.NormalizeReferences.Bindings)})
|
||||
}
|
||||
report.Steps = append(report.Steps, description)
|
||||
}
|
||||
for _, chain := range resolved.ValidatorChains {
|
||||
description := pipelineValidatorDescription{Stage: chain.Stage, Lane: chain.LaneID, Module: chain.ModuleKey, Validators: make([]pipelineModuleDescription, 0, len(chain.Validators))}
|
||||
for _, validator := range chain.Validators {
|
||||
description.Validators = append(description.Validators, module(validator.Binding, validator.ExecutionClass, validator.Prompts, nil, nil))
|
||||
}
|
||||
report.Validators = append(report.Validators, description)
|
||||
}
|
||||
return report
|
||||
}
|
||||
120
internal/preflight/preflight.go
Normal file
120
internal/preflight/preflight.go
Normal file
@@ -0,0 +1,120 @@
|
||||
// Package preflight checks statically known native resources for an already
|
||||
// resolved pipeline. Acquisition, construction, and presentation belong to callers.
|
||||
package preflight
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
// RequiresEngine reports selected model work, including nested validators.
|
||||
// Deterministic pipelines need no engine merely to validate their composition.
|
||||
func RequiresEngine(resolved pipeline.ResolvedPipeline) bool {
|
||||
if resolved.InputExecutionClass == contracts.ExecutionClassLLMBacked || resolved.ChunkExecutionClass == contracts.ExecutionClassLLMBacked || resolved.OutputExecutionClass == contracts.ExecutionClassLLMBacked {
|
||||
return true
|
||||
}
|
||||
for _, step := range resolved.Steps {
|
||||
for _, lane := range step.ArtifactLanes {
|
||||
if lane.ExtractExecutionClass == contracts.ExecutionClassLLMBacked || lane.MergeExecutionClass == contracts.ExecutionClassLLMBacked || lane.NormalizeExecutionClass == contracts.ExecutionClassLLMBacked {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, chain := range resolved.ValidatorChains {
|
||||
for _, validator := range chain.Validators {
|
||||
if validator.ExecutionClass == contracts.ExecutionClassLLMBacked {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Validate checks the same declarations and resource selections used by native
|
||||
// request builders. It does not render, prepare, read input/reference files,
|
||||
// resolve credential values, or call providers. Dynamic selections remain runtime
|
||||
// checks. Callers first use their ordinary application resolver for module,
|
||||
// options, type, validator-chain, and ordered-reference compatibility.
|
||||
func Validate(ctx context.Context, resolved pipeline.ResolvedPipeline, catalog pipeline.ModuleCatalog, engine *promptkit.Engine) error {
|
||||
if ctx == nil {
|
||||
return errors.New("preflight context must not be nil")
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
var findings []error
|
||||
check := func(stage pipeline.ModuleStage, step, lane, module, validator string, class contracts.ExecutionClass, profile string, prompts []string) {
|
||||
if class != contracts.ExecutionClassLLMBacked || ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
scope := fmt.Sprintf("pipeline %q", resolved.ID)
|
||||
if step != "" {
|
||||
scope += fmt.Sprintf(" step %q", step)
|
||||
}
|
||||
if lane != "" {
|
||||
scope += fmt.Sprintf(" lane %q", lane)
|
||||
}
|
||||
scope += fmt.Sprintf(" %s module %q", stage, module)
|
||||
if validator != "" {
|
||||
scope += fmt.Sprintf(" validator %q", validator)
|
||||
}
|
||||
if engine == nil {
|
||||
findings = append(findings, fmt.Errorf("%s: %w: native engine is required", scope, promptkit.ErrInvalidConfig))
|
||||
return
|
||||
}
|
||||
if profile != "" {
|
||||
if _, err := engine.InspectProfile(ctx, profile); err != nil {
|
||||
findings = append(findings, fmt.Errorf("%s profile %q: %w", scope, profile, err))
|
||||
}
|
||||
}
|
||||
for _, id := range prompts {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
prompt, err := engine.InspectPrompt(ctx, id)
|
||||
if err != nil {
|
||||
findings = append(findings, fmt.Errorf("%s prompt %q: %w", scope, id, err))
|
||||
continue
|
||||
}
|
||||
if profile != "" {
|
||||
continue
|
||||
}
|
||||
if prompt.DefaultProfileID == nil || *prompt.DefaultProfileID == "" {
|
||||
// A module may select a profile dynamically when constructing its
|
||||
// actual request. No declared selection is not a missing resource.
|
||||
continue
|
||||
}
|
||||
if _, err := engine.InspectProfile(ctx, *prompt.DefaultProfileID); err != nil {
|
||||
findings = append(findings, fmt.Errorf("%s prompt %q default profile %q: %w", scope, id, *prompt.DefaultProfileID, err))
|
||||
}
|
||||
}
|
||||
}
|
||||
input, _ := catalog.Inputs.Spec(resolved.Input.Module)
|
||||
check(pipeline.StageInput, "", "", resolved.Input.Module, "", resolved.InputExecutionClass, resolved.Input.LLMProfile, input.Prompts)
|
||||
check(pipeline.StageChunk, "", "", resolved.Chunk.Module, "", resolved.ChunkExecutionClass, resolved.Chunk.LLMProfile, resolved.ChunkPrompts)
|
||||
steps := make(map[string]string)
|
||||
for _, step := range resolved.Steps {
|
||||
for _, lane := range step.ArtifactLanes {
|
||||
steps[lane.ID] = step.ID
|
||||
check(pipeline.StageExtract, step.ID, lane.ID, lane.Extract.Module, "", lane.ExtractExecutionClass, lane.Extract.LLMProfile, lane.ExtractPrompts)
|
||||
check(pipeline.StageMerge, step.ID, lane.ID, lane.Merge.Module, "", lane.MergeExecutionClass, lane.Merge.LLMProfile, lane.MergePrompts)
|
||||
check(pipeline.StageNormalize, step.ID, lane.ID, lane.Normalize.Module, "", lane.NormalizeExecutionClass, lane.Normalize.LLMProfile, lane.NormalizePrompts)
|
||||
}
|
||||
}
|
||||
for _, chain := range resolved.ValidatorChains {
|
||||
for _, validator := range chain.Validators {
|
||||
check(chain.Stage, steps[chain.LaneID], chain.LaneID, chain.ModuleKey, validator.Binding.Module, validator.ExecutionClass, validator.Binding.LLMProfile, validator.Prompts)
|
||||
}
|
||||
}
|
||||
output, _ := catalog.Outputs.Spec(resolved.Output.Module)
|
||||
check(pipeline.StageOutput, "", "", resolved.Output.Module, "", resolved.OutputExecutionClass, resolved.Output.LLMProfile, output.Prompts)
|
||||
if err := ctx.Err(); err != nil {
|
||||
findings = append(findings, err)
|
||||
}
|
||||
return errors.Join(findings...)
|
||||
}
|
||||
163
internal/preflight/preflight_test.go
Normal file
163
internal/preflight/preflight_test.go
Normal file
@@ -0,0 +1,163 @@
|
||||
package preflight
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"go.maximumdirect.net/configkit"
|
||||
"go.maximumdirect.net/promptkit"
|
||||
)
|
||||
|
||||
type noGeneration struct{ t *testing.T }
|
||||
|
||||
func (client noGeneration) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
client.t.Fatal("offline preflight invoked a provider")
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func inspectionEngine(t *testing.T) *promptkit.Engine {
|
||||
t.Helper()
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{promptkit.SettingsInput(configkit.Bytes("offline resources", []byte(`profiles:
|
||||
- id: partial
|
||||
model: fixture
|
||||
api_key_env: NOTARIUS_PREFLIGHT_MUST_NOT_REQUIRE_THIS_CREDENTIAL
|
||||
api_key_required: true
|
||||
prompts:
|
||||
- id: initial
|
||||
default_profile: partial
|
||||
inputs: [{name: source, required: true}]
|
||||
messages: [{role: user, content: '{{input "source"}}'}]
|
||||
output: {format: text, validation_mode: none}
|
||||
- id: correction
|
||||
default_profile: partial
|
||||
messages: [{role: user, content: 'Correct the previous response.'}]
|
||||
output: {format: text, validation_mode: none}
|
||||
- id: validator
|
||||
default_profile: partial
|
||||
messages: [{role: user, content: 'Assess.'}]
|
||||
output: {format: text, validation_mode: none}
|
||||
- id: unknown-default
|
||||
default_profile: missing-profile
|
||||
messages: [{role: user, content: 'Select a profile.'}]
|
||||
output: {format: text, validation_mode: none}
|
||||
- id: no-default
|
||||
messages: [{role: user, content: 'Select a profile.'}]
|
||||
output: {format: text, validation_mode: none}
|
||||
`)))}}}}}, promptkit.WithBundledOpenRouter(false), promptkit.WithLLMClient(noGeneration{t}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return engine
|
||||
}
|
||||
|
||||
func resolvedFixture() pipeline.ResolvedPipeline {
|
||||
return pipeline.ResolvedPipeline{ID: "campaign", Steps: []pipeline.ResolvedPipelineStep{{ID: "identify", ArtifactLanes: []pipeline.ResolvedArtifactLane{{ID: "characters", Extract: pipeline.Binding("character-producer"), ExtractExecutionClass: contracts.ExecutionClassLLMBacked, ExtractPrompts: []string{"initial", "correction"}}}}},
|
||||
ValidatorChains: []pipeline.ResolvedValidatorChain{{Stage: pipeline.StageExtract, LaneID: "characters", ModuleKey: "character-producer", Validators: []pipeline.ResolvedValidator{{Binding: pipeline.Binding("domain-validator"), ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{"validator"}}}}},
|
||||
}
|
||||
}
|
||||
|
||||
func TestOfflinePreflightAcceptsPartialDefaultsWithoutExecutionInputs(t *testing.T) {
|
||||
engine := inspectionEngine(t)
|
||||
resolved := resolvedFixture()
|
||||
resolved.Steps[0].ArtifactLanes[0].ExtractReferences.Bindings = []pipeline.ReferenceBinding{{Source: "/not-present/reference.json"}}
|
||||
if !RequiresEngine(resolved) {
|
||||
t.Fatal("selected LLM roles were missed")
|
||||
}
|
||||
if err := Validate(context.Background(), resolved, pipeline.ModuleCatalog{}, engine); err != nil {
|
||||
t.Fatalf("partial profile or absent execution inputs rejected: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOfflinePreflightReportsEveryKnownResourceWithScopeAndCause(t *testing.T) {
|
||||
resolved := resolvedFixture()
|
||||
lane := &resolved.Steps[0].ArtifactLanes[0]
|
||||
lane.Extract.LLMProfile = "missing-explicit"
|
||||
lane.ExtractPrompts = []string{"missing-initial", "missing-correction"}
|
||||
resolved.ValidatorChains[0].Validators[0].Prompts = []string{"missing-validator"}
|
||||
err := Validate(context.Background(), resolved, pipeline.ModuleCatalog{}, inspectionEngine(t))
|
||||
if !errors.Is(err, promptkit.ErrPromptNotFound) || !errors.Is(err, promptkit.ErrProfileNotFound) {
|
||||
t.Fatalf("native causes lost: %v", err)
|
||||
}
|
||||
for _, value := range []string{"campaign", "identify", "characters", "extract", "character-producer", "domain-validator", "missing-explicit", "missing-initial", "missing-correction", "missing-validator"} {
|
||||
if !strings.Contains(err.Error(), value) {
|
||||
t.Fatalf("missing preflight context %q: %v", value, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestOfflinePreflightChecksApplicableProfilesButNotExecutionReadiness(t *testing.T) {
|
||||
engine := inspectionEngine(t)
|
||||
for _, test := range []struct {
|
||||
prompt, profile string
|
||||
want error
|
||||
}{
|
||||
{"unknown-default", "", promptkit.ErrProfileNotFound},
|
||||
{"no-default", "", nil},
|
||||
{"unknown-default", "partial", nil},
|
||||
{"no-default", "partial", nil},
|
||||
} {
|
||||
resolved := resolvedFixture()
|
||||
lane := &resolved.Steps[0].ArtifactLanes[0]
|
||||
lane.ExtractPrompts, lane.Extract.LLMProfile = []string{test.prompt}, test.profile
|
||||
err := Validate(context.Background(), resolved, pipeline.ModuleCatalog{}, engine)
|
||||
if test.want == nil && err != nil || test.want != nil && !errors.Is(err, test.want) {
|
||||
t.Fatalf("prompt=%q profile=%q: %v", test.prompt, test.profile, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestOfflinePreflightLeavesDynamicSelectionsForRuntime(t *testing.T) {
|
||||
resolved := resolvedFixture()
|
||||
resolved.Steps[0].ArtifactLanes[0].ExtractPrompts = nil
|
||||
resolved.ValidatorChains = nil
|
||||
if err := Validate(context.Background(), resolved, pipeline.ModuleCatalog{}, inspectionEngine(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOfflinePreflightIncludesInputOutputAndNestedValidatorsWithoutConstruction(t *testing.T) {
|
||||
catalog := pipeline.ModuleCatalog{Inputs: pipeline.NewInputAdapterRegistry(), Outputs: pipeline.NewOutputEncoderRegistry()}
|
||||
for _, stage := range []pipeline.ModuleStage{pipeline.StageInput, pipeline.StageOutput} {
|
||||
spec := pipeline.ModuleSpec{Key: string(stage), Stage: stage, ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{"missing-" + string(stage)}}
|
||||
var err error
|
||||
if stage == pipeline.StageInput {
|
||||
err = catalog.Inputs.RegisterWithSpec(spec, func() (contracts.InputAdapter, error) { t.Fatal("preflight constructed input"); return nil, nil })
|
||||
} else {
|
||||
err = catalog.Outputs.RegisterWithSpec(spec, func() (contracts.OutputEncoder, error) { t.Fatal("preflight constructed output"); return nil, nil })
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
resolved := pipeline.ResolvedPipeline{ID: "io", Input: pipeline.Binding("input"), InputExecutionClass: contracts.ExecutionClassLLMBacked, Output: pipeline.Binding("output"), OutputExecutionClass: contracts.ExecutionClassLLMBacked}
|
||||
err := Validate(context.Background(), resolved, catalog, inspectionEngine(t))
|
||||
if !errors.Is(err, promptkit.ErrPromptNotFound) || !strings.Contains(err.Error(), "missing-input") || !strings.Contains(err.Error(), "missing-output") {
|
||||
t.Fatalf("input/output declarations were missed: %v", err)
|
||||
}
|
||||
resolved = resolvedFixture()
|
||||
resolved.Steps[0].ArtifactLanes[0].ExtractExecutionClass = contracts.ExecutionClassDeterministic
|
||||
if !RequiresEngine(resolved) {
|
||||
t.Fatal("nested validator omitted from engine selection")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeterministicPreflightNeedsNoEngineAndCancellationStopsInspection(t *testing.T) {
|
||||
if RequiresEngine(pipeline.ResolvedPipeline{}) {
|
||||
t.Fatal("deterministic composition requires engine")
|
||||
}
|
||||
if err := Validate(context.Background(), pipeline.ResolvedPipeline{}, pipeline.ModuleCatalog{}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := Validate(context.Background(), resolvedFixture(), pipeline.ModuleCatalog{}, nil); !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||
t.Fatalf("missing engine: %v", err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if err := Validate(ctx, resolvedFixture(), pipeline.ModuleCatalog{}, nil); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("cancellation lost: %v", err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user