Add offline pipeline inspection and shared native resource preflight

This commit is contained in:
2026-11-22 03:32:31 +00:00
parent 9f66421f4f
commit 2a9c5a6075
9 changed files with 806 additions and 10 deletions

View File

@@ -2,10 +2,9 @@ package cli
import (
"context"
"fmt"
"strings"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"go.maximumdirect.net/ckconfig"
"go.maximumdirect.net/ckprompt"
"go.maximumdirect.net/commandkit"
@@ -42,6 +41,7 @@ func NewApplication(version string, options Options) (*commandkit.App, error) {
definitions = ckprompt.WithNamespaces(definitions, promptkit.Namespaces{})
declarations := []commandkit.Declaration{
commandkit.Completion("notarius"),
pipelineCommands(newState, func() (pipeline.ModuleCatalog, error) { return effectiveCatalog(options) }),
ckconfig.Command(ckconfig.Suite[struct{}, *nativeLoaders]{
NewState: newState,
CheckInput: func(_ string, selection ckconfig.Selection, _ struct{}) error {
@@ -53,10 +53,3 @@ func NewApplication(version string, options Options) (*commandkit.App, error) {
declarations = append(declarations, ckprompt.Declarations(definitions)...)
return commandkit.New(commandkit.Identity{Name: "notarius", Version: version}, declarations...)
}
func checkConfigSelection(selection ckconfig.Selection) error {
if selection.Config.Set && strings.TrimSpace(selection.Config.Value) == "" {
return fmt.Errorf("--config must not be empty")
}
return nil
}

View File

@@ -17,7 +17,7 @@ func TestNativeApplicationBuiltinsNeedNoConfiguration(t *testing.T) {
if err != nil {
t.Fatal(err)
}
for _, args := range [][]string{{"--help"}, {"help"}, {"version"}, {"completion", "bash"}, {"completion", "zsh"}, {"completion", "fish"}, {"config", "--help"}} {
for _, args := range [][]string{{"--help"}, {"help"}, {"version"}, {"completion", "bash"}, {"completion", "zsh"}, {"completion", "fish"}, {"config", "--help"}, {"pipeline", "--help"}, {"pipeline", "validate", "--help"}} {
var out, diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &out, Err: &diagnostics})
if outcome.Status != 0 || out.Len() == 0 {
@@ -52,6 +52,7 @@ func TestNativeConfigurationScopesAndDiffUseSelectedLoadEvidence(t *testing.T) {
{"differences", []string{"config", "diff", "--config", left, "--against-config", right, "--check"}, 3, "workers"},
{"identical", []string{"config", "diff", "--config", left, "--against-config", left, "--check"}, 0, ""},
{"empty", []string{"config", "show", "--config", ""}, 2, ""},
{"configuration has no pipeline mode", []string{"config", "validate", "--config", left, "--pipeline", "demo"}, 2, ""},
} {
t.Run(test.name, func(t *testing.T) {
var out, diagnostics bytes.Buffer

View File

@@ -31,6 +31,13 @@ type loadedSelection struct {
engine *promptkit.Engine
}
func checkConfigSelection(selection ckconfig.Selection) error {
if selection.Config.Set && strings.TrimSpace(selection.Config.Value) == "" {
return fmt.Errorf("--config must not be empty")
}
return nil
}
func (loaders *nativeLoaders) application(ctx context.Context, selection ckconfig.Selection) (*configkit.Result[config.Document], error) {
if err := ctx.Err(); err != nil {
return nil, err

View File

@@ -0,0 +1,143 @@
package cli
import (
"context"
"errors"
"fmt"
"sort"
"strings"
"text/template"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"gitea.maximumdirect.net/eric/notarius/internal/preflight"
"go.maximumdirect.net/ckconfig"
"go.maximumdirect.net/commandkit"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
type pipelineInspectionInput struct {
Selection ckconfig.Selection
PipelineID string
}
func configurationSelectionFlags[I any](destination func(*I) *ckconfig.Selection) []commandkit.Flag[I] {
return []commandkit.Flag[I]{
commandkit.ValueFlag(commandkit.FlagMeta{Name: "config", Usage: "Select the application configuration.", Completion: commandkit.CompleteFiles()}, (*string)(nil), func(value string) (string, error) {
if strings.TrimSpace(value) == "" {
return "", fmt.Errorf("--config must not be empty")
}
return value, nil
}, func(input *I, value commandkit.Supplied[string]) { destination(input).Config = value }),
commandkit.ValueFlag(commandkit.FlagMeta{Name: "overlay", Usage: "Select a configuration overlay.", Completion: commandkit.CompleteFiles()}, (*string)(nil), func(value string) (string, error) { return value, nil }, func(input *I, value commandkit.Supplied[string]) { destination(input).Overlay = value }),
commandkit.RepeatFlag(commandkit.FlagMeta{Name: "set", Usage: "Override POINTER=JSON."}, []configkit.Override(nil), parseConfigurationOverride, func(input *I, value commandkit.Supplied[[]configkit.Override]) {
destination(input).Overrides = value.Value
}),
}
}
func parseConfigurationOverride(raw string) (configkit.Override, error) {
pointer, encoded, ok := strings.Cut(raw, "=")
if !ok {
return configkit.Override{}, fmt.Errorf("override requires POINTER=JSON")
}
path, err := configkit.ParsePath(pointer)
if err != nil {
return configkit.Override{}, err
}
value, err := commandkit.ParseCLIJSON(encoded)
if err != nil {
return configkit.Override{}, err
}
return configkit.Override{Path: path, Value: value, Flag: "--set"}, nil
}
func pipelineCommands(newState func() *nativeLoaders, catalog func() (pipeline.ModuleCatalog, error)) commandkit.Command {
command := func(operation, summary string) commandkit.Command {
var arguments []commandkit.Argument[pipelineInspectionInput]
if operation != "list" {
arguments = []commandkit.Argument[pipelineInspectionInput]{commandkit.ValueArgument(commandkit.ArgumentMeta{Name: "pipeline-id", Usage: "Configured pipeline identifier."}, true, func(value string) (string, error) {
value = strings.TrimSpace(value)
if value == "" {
return "", fmt.Errorf("pipeline id must not be empty")
}
return value, nil
}, func(input *pipelineInspectionInput, value commandkit.Supplied[string]) {
input.PipelineID = value.Value
})}
}
return commandkit.NewCommand(commandkit.CommandSpec[pipelineInspectionInput]{Name: operation, Summary: summary,
Flags: configurationSelectionFlags(func(input *pipelineInspectionInput) *ckconfig.Selection { return &input.Selection }), Arguments: arguments,
CheckInput: func(input pipelineInspectionInput) error { return checkConfigSelection(input.Selection) },
Run: func(invocation commandkit.Invocation, input pipelineInspectionInput) commandkit.Outcome {
state := newState()
result, err := state.application(invocation.Context, input.Selection)
if err != nil {
return pipelineConfigurationFailure(err)
}
if operation == "list" {
ids := make([]string, 0, len(result.Config.Pipelines))
for id := range result.Config.Pipelines {
ids = append(ids, id)
}
sort.Strings(ids)
return reportPipelineInspection(invocation, commandkit.Report{Value: ids})
}
loaded, err := config.FromResult(result)
if err != nil {
return pipelineInvalidConfiguration(err)
}
modules, err := catalog()
if err != nil {
return commandkit.Failure(err.Error()).WithCause(err)
}
effective, err := loaded.RuntimeConfig().Resolve(config.ResolveInput{PipelineID: input.PipelineID, Catalog: modules})
if err != nil {
return pipelineInvalidConfiguration(err)
}
if operation == "show" {
return reportPipelineInspection(invocation, commandkit.Report{Value: describePipeline(effective.ResolvedPipeline, modules)})
}
var engine *promptkit.Engine
if preflight.RequiresEngine(effective.ResolvedPipeline) {
engine, err = state.engine(invocation.Context, input.Selection)
if err != nil {
return pipelineConfigurationFailure(fmt.Errorf("pipeline %q native resources: %w", input.PipelineID, err))
}
}
if err := preflight.Validate(invocation.Context, effective.ResolvedPipeline, modules, engine); err != nil {
return pipelineInvalidConfiguration(err)
}
return reportPipelineInspection(invocation, commandkit.Report{Value: struct {
Pipeline string `json:"pipeline"`
Valid bool `json:"valid"`
}{input.PipelineID, true}, TextTemplate: pipelineValidationTemplate})
},
})
}
return commandkit.Group("pipeline", "Inspect, validate, or execute configured pipelines.", command("list", "List configured pipeline identifiers."), command("show", "Show a resolved pipeline composition."), command("validate", "Validate known pipeline dependencies offline."))
}
var pipelineValidationTemplate = template.Must(commandkit.ParseTextTemplate("pipeline-validation", "Pipeline {{printf \"%q\" .Pipeline}} is valid.\n", nil))
func reportPipelineInspection(invocation commandkit.Invocation, report commandkit.Report) commandkit.Outcome {
if err := invocation.Reporter().Report(report); err != nil {
return commandkit.Failure("Could not report pipeline inspection.").WithCause(err)
}
return commandkit.Success()
}
func pipelineConfigurationFailure(err error) commandkit.Outcome {
if ckconfig.ClassifyConfigurationError(err) == commandkit.OutcomeUsage {
return commandkit.Usage(err.Error()).WithCause(err)
}
return commandkit.Failure(err.Error()).WithCause(err)
}
func pipelineInvalidConfiguration(err error) commandkit.Outcome {
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return commandkit.Failure(err.Error()).WithCause(err)
}
return commandkit.Usage(err.Error()).WithCause(err)
}

View File

@@ -0,0 +1,276 @@
package cli
import (
"bytes"
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"go.maximumdirect.net/commandkit"
"go.maximumdirect.net/promptkit"
)
const inspectionConfiguration = `version: 6
output: {directory: output-must-not-exist}
cache_directory: cache-must-not-exist
debug: {directory: debug-must-not-exist}
pipelines:
demo:
input: {module: fixture/input}
chunk: {module: fixture/chunk}
output: {module: fixture/output}
steps:
- id: build
artifacts:
notes:
extract:
module: fixture/extract
references: {context: {path: reference-must-not-exist.txt}}
validators: [{module: fixture/validator}]
merge: {module: fixture/merge}
normalize: {module: fixture/normalize}
promptkit:
profiles:
- id: partial
model: fixture-model
api_key_env: NOTARIUS_INSPECTION_NO_CREDENTIAL_REQUIRED
api_key_required: true
prompts:
- id: inspect-initial
default_profile: partial
inputs: [{name: source, required: true}]
messages: [{role: user, content: '{{input "source"}}'}]
output: {format: text, validation_mode: none}
- id: inspect-correction
default_profile: partial
messages: [{role: user, content: Correct.}]
output: {format: text, validation_mode: none}
- id: inspect-validator
default_profile: partial
messages: [{role: user, content: Assess.}]
output: {format: text, validation_mode: none}
`
type inspectionArtifact struct {
Value string `json:"value"`
}
type inspectionCodec struct{}
func (inspectionCodec) Kind() contracts.ArtifactKind { return "test/inspection" }
func (inspectionCodec) MediaType() string { return "application/json" }
func (inspectionCodec) Schema() contracts.ArtifactSchema {
return contracts.ArtifactSchema{ID: "test.inspection", Name: "inspection", Version: "v1", JSONSchema: []byte(`{"type":"object","properties":{"value":{"type":"string"}},"required":["value"]}`)}
}
func (inspectionCodec) Encode(value inspectionArtifact) ([]byte, error) { return json.Marshal(value) }
func (codec inspectionCodec) EncodeCandidate(value inspectionArtifact) ([]byte, error) {
return codec.Encode(value)
}
func (codec inspectionCodec) DecodeCandidate(content []byte) (inspectionArtifact, error) {
return codec.Decode(content)
}
func (inspectionCodec) Decode(content []byte) (inspectionArtifact, error) {
var value inspectionArtifact
err := json.Unmarshal(content, &value)
return value, err
}
func inspectionCatalog(t *testing.T, class contracts.ExecutionClass) pipeline.ModuleCatalog {
t.Helper()
catalog := pipeline.ModuleCatalog{Inputs: pipeline.NewInputAdapterRegistry(), Chunkers: pipeline.NewChunkerRegistry(), ArtifactCodecs: pipeline.NewArtifactCodecRegistry(), Extractors: pipeline.NewExtractorRegistry(), Mergers: pipeline.NewMergerRegistry(), Normalizers: pipeline.NewNormalizerRegistry(), Validators: pipeline.NewValidatorRegistry(), ValidatorChains: pipeline.NewValidatorChainRegistry(), Outputs: pipeline.NewOutputEncoderRegistry()}
check := func(err error) {
t.Helper()
if err != nil {
t.Fatal(err)
}
}
spec := func(stage pipeline.ModuleStage, key string) pipeline.ModuleSpec {
return pipeline.ModuleSpec{Key: key, Stage: stage, ExecutionClass: contracts.ExecutionClassDeterministic}
}
check(catalog.Inputs.RegisterWithSpec(spec(pipeline.StageInput, "fixture/input"), func() (contracts.InputAdapter, error) { t.Fatal("inspection constructed input"); return nil, nil }))
check(catalog.Chunkers.RegisterWithSpec(spec(pipeline.StageChunk, "fixture/chunk"), func() (contracts.Chunker, error) { t.Fatal("inspection constructed chunker"); return nil, nil }))
check(catalog.Outputs.RegisterWithSpec(spec(pipeline.StageOutput, "fixture/output"), func() (contracts.OutputEncoder, error) { t.Fatal("inspection constructed output"); return nil, nil }))
check(pipeline.RegisterArtifactCodec(catalog.ArtifactCodecs, inspectionCodec{}))
extract := spec(pipeline.StageExtract, "fixture/extract")
extract.ArtifactKind, extract.ExecutionClass, extract.Prompts = "test/inspection", class, []string{"inspect-initial", "inspect-correction"}
extract.ReferenceSlots = []contracts.ReferenceSlot{{Name: "context", Required: true, AcceptedMediaTypes: []string{"text/plain"}}}
check(pipeline.RegisterExtractor(catalog.Extractors, extract, func() (contracts.Extractor[inspectionArtifact], error) {
t.Fatal("inspection constructed extractor")
return nil, nil
}))
merge := spec(pipeline.StageMerge, "fixture/merge")
merge.ArtifactKind = "test/inspection"
check(pipeline.RegisterMerger(catalog.Mergers, merge, func() (contracts.Merger[inspectionArtifact], error) {
t.Fatal("inspection constructed merger")
return nil, nil
}))
normalize := spec(pipeline.StageNormalize, "fixture/normalize")
normalize.ArtifactKind = "test/inspection"
check(pipeline.RegisterNormalizer(catalog.Normalizers, normalize, func() (contracts.Normalizer[inspectionArtifact], error) {
t.Fatal("inspection constructed normalizer")
return nil, nil
}))
check(pipeline.RegisterTypedValidator(catalog.Validators, "test/inspection", pipeline.ValidatorSpec{Key: "fixture/validator", ExecutionClass: class, Prompts: []string{"inspect-validator"}}, func() (contracts.TypedValidator[inspectionArtifact], error) {
t.Fatal("inspection constructed validator")
return nil, nil
}))
return catalog
}
type inspectionProvider struct{ t *testing.T }
func (provider inspectionProvider) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
provider.t.Fatal("inspection called provider")
return nil, nil
}
func writeInspectionConfiguration(t *testing.T, content string) string {
t.Helper()
file := filepath.Join(t.TempDir(), "config.yml")
if err := os.WriteFile(file, []byte(content), 0600); err != nil {
t.Fatal(err)
}
return file
}
func newInspectionApplication(t *testing.T, catalog func() (pipeline.ModuleCatalog, error)) (*commandkit.App, **nativeLoaders) {
t.Helper()
var last *nativeLoaders
app, err := commandkit.New(commandkit.Identity{Name: "notarius"}, pipelineCommands(func() *nativeLoaders {
last = &nativeLoaders{engineOptions: []promptkit.Option{promptkit.WithLLMClient(inspectionProvider{t})}}
return last
}, catalog))
if err != nil {
t.Fatal(err)
}
return app, &last
}
func TestPipelineListOnlyEnumeratesCapturedConfiguration(t *testing.T) {
content := strings.Replace(inspectionConfiguration, " demo:", " zeta:", 1)
content = strings.Replace(content, "promptkit:", " alpha:\n input: {module: unavailable}\n artifacts:\n value:\n extract: {module: unavailable}\npromptkit:", 1)
file := writeInspectionConfiguration(t, content)
app, state := newInspectionApplication(t, func() (pipeline.ModuleCatalog, error) {
t.Fatal("list constructed a catalog")
return pipeline.ModuleCatalog{}, nil
})
var out, diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), []string{"pipeline", "list", "--config", file, "--format", "json"}, commandkit.Streams{Out: &out, Err: &diagnostics})
var ids []string
if outcome.Status != 0 || json.Unmarshal(out.Bytes(), &ids) != nil || !reflect.DeepEqual(ids, []string{"alpha", "zeta"}) {
t.Fatalf("list=%#v %s %s", outcome, out.String(), diagnostics.String())
}
if len((*state).selections) != 1 || (*state).selections[0].engine != nil {
t.Fatal("list expanded native resources")
}
}
func TestPipelineShowResolvesCompositionWithoutEngineOrExecutionInputs(t *testing.T) {
file := writeInspectionConfiguration(t, inspectionConfiguration)
catalog := inspectionCatalog(t, contracts.ExecutionClassLLMBacked)
app, state := newInspectionApplication(t, func() (pipeline.ModuleCatalog, error) { return catalog, nil })
var out, diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), []string{"pipeline", "show", "demo", "--config", file, "--format", "json"}, commandkit.Streams{Out: &out, Err: &diagnostics})
var report pipelineDescription
if err := json.Unmarshal(out.Bytes(), &report); err != nil || outcome.Status != 0 {
t.Fatalf("show=%#v %s %s %v", outcome, out.String(), diagnostics.String(), err)
}
if report.Pipeline != "demo" || len(report.Steps) != 1 || report.Steps[0].ID != "build" || len(report.Steps[0].Lanes) != 1 || report.Steps[0].Lanes[0].Extract.Module != "fixture/extract" || len(report.Steps[0].Lanes[0].Extract.References) != 1 || len(report.Validators) == 0 {
t.Fatalf("missing resolved composition: %#v", report)
}
if (*state).selections[0].engine != nil {
t.Fatal("show expanded profiles by constructing engine")
}
assertInspectionNoRunStorage(t, file)
}
func TestPipelineValidateChecksNativeDeclarationsOfflineAndUsesOneLoad(t *testing.T) {
for _, class := range []contracts.ExecutionClass{contracts.ExecutionClassDeterministic, contracts.ExecutionClassLLMBacked} {
file := writeInspectionConfiguration(t, inspectionConfiguration)
catalog := inspectionCatalog(t, class)
app, state := newInspectionApplication(t, func() (pipeline.ModuleCatalog, error) { return catalog, nil })
var out, diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), []string{"pipeline", "validate", "demo", "--config", file}, commandkit.Streams{Out: &out, Err: &diagnostics})
if outcome.Status != 0 || !strings.Contains(out.String(), "demo") {
t.Fatalf("validate=%#v %s %s", outcome, out.String(), diagnostics.String())
}
if len((*state).selections) != 1 || ((*state).selections[0].engine != nil) != (class == contracts.ExecutionClassLLMBacked) {
t.Fatal("validation loaded twice or built an unnecessary engine")
}
assertInspectionNoRunStorage(t, file)
}
}
func TestPipelineValidateRejectsKnownMissingResourcesWithScope(t *testing.T) {
for _, id := range []string{"inspect-initial", "inspect-correction", "inspect-validator"} {
file := writeInspectionConfiguration(t, strings.Replace(inspectionConfiguration, "id: "+id, "id: unused-"+id, 1))
catalog := inspectionCatalog(t, contracts.ExecutionClassLLMBacked)
app, _ := newInspectionApplication(t, func() (pipeline.ModuleCatalog, error) { return catalog, nil })
var out, diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), []string{"pipeline", "validate", "demo", "--config", file}, commandkit.Streams{Out: &out, Err: &diagnostics})
if outcome.Status != 2 || !errors.Is(outcome.Cause, promptkit.ErrPromptNotFound) || !strings.Contains(diagnostics.String(), id) || !strings.Contains(diagnostics.String(), "build") || !strings.Contains(diagnostics.String(), "notes") {
t.Fatalf("known dependency=%#v %s", outcome, diagnostics.String())
}
}
}
func TestPipelineInspectionUsesNativeArgumentsSelectionsAndDelivery(t *testing.T) {
file := writeInspectionConfiguration(t, inspectionConfiguration)
catalog := inspectionCatalog(t, contracts.ExecutionClassLLMBacked)
app, _ := newInspectionApplication(t, func() (pipeline.ModuleCatalog, error) { return catalog, nil })
for _, test := range []struct {
args []string
status int
}{
{[]string{"pipeline", "show"}, 2},
{[]string{"pipeline", "show", "demo", "--config", "", "--config", file}, 2},
{[]string{"pipeline", "show", "unknown", "--config", file}, 2},
{[]string{"pipeline", "show", "demo", "--config", file, "--set", `/pipelines/demo/steps/0/artifacts/notes/extract/options={"unsupported":true}`}, 2},
{[]string{"pipeline", "validate", "demo", "--config", file, "--set", `/pipelines/demo/llm_profile="missing-profile"`}, 2},
{[]string{"pipeline", "show", "demo", "--config", file, "--set", `/pipelines/demo/steps/0/artifacts/notes/extract/references=null`}, 2},
{[]string{"pipeline", "list", "--config", file, "--set", `/pipelines/demo/input/module="another-input"`}, 0},
{[]string{"pipeline", "validate", "demo", "--config", file, "--quiet"}, 0},
{[]string{"pipeline", "--help"}, 0},
} {
var out, diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), test.args, commandkit.Streams{Out: &out, Err: &diagnostics})
if outcome.Status != test.status {
t.Fatalf("%v: %#v %s", test.args, outcome, diagnostics.String())
}
if test.args[len(test.args)-1] == "--quiet" && out.Len() != 0 {
t.Fatal("quiet pipeline inspection wrote stdout")
}
}
}
type inspectionFailWriter struct{ err error }
func (writer inspectionFailWriter) Write([]byte) (int, error) { return 0, writer.err }
func TestPipelineInspectionReportFailureRemainsOperational(t *testing.T) {
file := writeInspectionConfiguration(t, inspectionConfiguration)
app, _ := newInspectionApplication(t, func() (pipeline.ModuleCatalog, error) {
t.Fatal("list loaded catalog")
return pipeline.ModuleCatalog{}, nil
})
primary := errors.New("inspection stdout delivery failed")
var diagnostics bytes.Buffer
outcome := app.Execute(context.Background(), []string{"pipeline", "list", "--config", file}, commandkit.Streams{Out: inspectionFailWriter{primary}, Err: &diagnostics})
if outcome.Status != 1 || !errors.Is(outcome.Cause, primary) {
t.Fatalf("delivery failure classified as invalid configuration: %#v %s", outcome, diagnostics.String())
}
}
func assertInspectionNoRunStorage(t *testing.T, file string) {
t.Helper()
entries, err := os.ReadDir(filepath.Dir(file))
if err != nil || len(entries) != 1 || entries[0].Name() != "config.yml" {
t.Fatalf("inspection read/materialized inputs or created storage: %v %v", entries, err)
}
}

View File

@@ -0,0 +1,72 @@
package cli
import (
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
// Inspection reports identities and resolved composition, not another effective
// configuration or profile expansion. ConfigKit owns secret-aware settings views.
type pipelineModuleDescription struct {
Module string `json:"module"`
ExecutionClass contracts.ExecutionClass `json:"execution_class"`
LLMProfile string `json:"llm_profile,omitempty"`
Controls *contracts.ProducerControls `json:"controls,omitempty"`
Prompts []string `json:"prompts,omitempty"`
References []pipeline.ReferenceBinding `json:"references,omitempty"`
}
type pipelineLaneDescription struct {
ID string `json:"id"`
ArtifactKind contracts.ArtifactKind `json:"artifact_kind"`
Extract pipelineModuleDescription `json:"extract"`
Merge pipelineModuleDescription `json:"merge"`
Normalize pipelineModuleDescription `json:"normalize"`
}
type pipelineStepDescription struct {
ID string `json:"id"`
Lanes []pipelineLaneDescription `json:"lanes"`
}
type pipelineValidatorDescription struct {
Stage pipeline.ModuleStage `json:"stage"`
Lane string `json:"lane,omitempty"`
Module string `json:"module"`
Validators []pipelineModuleDescription `json:"validators"`
}
type pipelineDescription struct {
Pipeline string `json:"pipeline"`
Input pipelineModuleDescription `json:"input"`
Chunk pipelineModuleDescription `json:"chunk"`
Steps []pipelineStepDescription `json:"steps"`
Validators []pipelineValidatorDescription `json:"validators,omitempty"`
Output pipelineModuleDescription `json:"output"`
}
func describePipeline(resolved pipeline.ResolvedPipeline, catalog pipeline.ModuleCatalog) pipelineDescription {
module := func(binding pipeline.ModuleBinding, class contracts.ExecutionClass, prompts []string, controls *contracts.ProducerControls, references []pipeline.ReferenceBinding) pipelineModuleDescription {
return pipelineModuleDescription{Module: binding.Module, ExecutionClass: class, LLMProfile: binding.LLMProfile, Prompts: prompts, Controls: controls, References: references}
}
input, _ := catalog.Inputs.Spec(resolved.Input.Module)
output, _ := catalog.Outputs.Spec(resolved.Output.Module)
report := pipelineDescription{Pipeline: resolved.ID, Input: module(resolved.Input, resolved.InputExecutionClass, input.Prompts, nil, nil),
Chunk: module(resolved.Chunk, resolved.ChunkExecutionClass, resolved.ChunkPrompts, &resolved.ChunkControls, resolved.ChunkReferences.Bindings), Output: module(resolved.Output, resolved.OutputExecutionClass, output.Prompts, nil, nil)}
for _, step := range resolved.Steps {
description := pipelineStepDescription{ID: step.ID, Lanes: make([]pipelineLaneDescription, 0, len(step.ArtifactLanes))}
for _, lane := range step.ArtifactLanes {
description.Lanes = append(description.Lanes, pipelineLaneDescription{ID: lane.ID, ArtifactKind: lane.ArtifactKind,
Extract: module(lane.Extract, lane.ExtractExecutionClass, lane.ExtractPrompts, &lane.ExtractControls, lane.ExtractReferences.Bindings),
Merge: module(lane.Merge, lane.MergeExecutionClass, lane.MergePrompts, &lane.MergeControls, lane.MergeReferences.Bindings),
Normalize: module(lane.Normalize, lane.NormalizeExecutionClass, lane.NormalizePrompts, &lane.NormalizeControls, lane.NormalizeReferences.Bindings)})
}
report.Steps = append(report.Steps, description)
}
for _, chain := range resolved.ValidatorChains {
description := pipelineValidatorDescription{Stage: chain.Stage, Lane: chain.LaneID, Module: chain.ModuleKey, Validators: make([]pipelineModuleDescription, 0, len(chain.Validators))}
for _, validator := range chain.Validators {
description.Validators = append(description.Validators, module(validator.Binding, validator.ExecutionClass, validator.Prompts, nil, nil))
}
report.Validators = append(report.Validators, description)
}
return report
}

View File

@@ -0,0 +1,120 @@
// Package preflight checks statically known native resources for an already
// resolved pipeline. Acquisition, construction, and presentation belong to callers.
package preflight
import (
"context"
"errors"
"fmt"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"go.maximumdirect.net/promptkit"
)
// RequiresEngine reports selected model work, including nested validators.
// Deterministic pipelines need no engine merely to validate their composition.
func RequiresEngine(resolved pipeline.ResolvedPipeline) bool {
if resolved.InputExecutionClass == contracts.ExecutionClassLLMBacked || resolved.ChunkExecutionClass == contracts.ExecutionClassLLMBacked || resolved.OutputExecutionClass == contracts.ExecutionClassLLMBacked {
return true
}
for _, step := range resolved.Steps {
for _, lane := range step.ArtifactLanes {
if lane.ExtractExecutionClass == contracts.ExecutionClassLLMBacked || lane.MergeExecutionClass == contracts.ExecutionClassLLMBacked || lane.NormalizeExecutionClass == contracts.ExecutionClassLLMBacked {
return true
}
}
}
for _, chain := range resolved.ValidatorChains {
for _, validator := range chain.Validators {
if validator.ExecutionClass == contracts.ExecutionClassLLMBacked {
return true
}
}
}
return false
}
// Validate checks the same declarations and resource selections used by native
// request builders. It does not render, prepare, read input/reference files,
// resolve credential values, or call providers. Dynamic selections remain runtime
// checks. Callers first use their ordinary application resolver for module,
// options, type, validator-chain, and ordered-reference compatibility.
func Validate(ctx context.Context, resolved pipeline.ResolvedPipeline, catalog pipeline.ModuleCatalog, engine *promptkit.Engine) error {
if ctx == nil {
return errors.New("preflight context must not be nil")
}
if err := ctx.Err(); err != nil {
return err
}
var findings []error
check := func(stage pipeline.ModuleStage, step, lane, module, validator string, class contracts.ExecutionClass, profile string, prompts []string) {
if class != contracts.ExecutionClassLLMBacked || ctx.Err() != nil {
return
}
scope := fmt.Sprintf("pipeline %q", resolved.ID)
if step != "" {
scope += fmt.Sprintf(" step %q", step)
}
if lane != "" {
scope += fmt.Sprintf(" lane %q", lane)
}
scope += fmt.Sprintf(" %s module %q", stage, module)
if validator != "" {
scope += fmt.Sprintf(" validator %q", validator)
}
if engine == nil {
findings = append(findings, fmt.Errorf("%s: %w: native engine is required", scope, promptkit.ErrInvalidConfig))
return
}
if profile != "" {
if _, err := engine.InspectProfile(ctx, profile); err != nil {
findings = append(findings, fmt.Errorf("%s profile %q: %w", scope, profile, err))
}
}
for _, id := range prompts {
if ctx.Err() != nil {
return
}
prompt, err := engine.InspectPrompt(ctx, id)
if err != nil {
findings = append(findings, fmt.Errorf("%s prompt %q: %w", scope, id, err))
continue
}
if profile != "" {
continue
}
if prompt.DefaultProfileID == nil || *prompt.DefaultProfileID == "" {
// A module may select a profile dynamically when constructing its
// actual request. No declared selection is not a missing resource.
continue
}
if _, err := engine.InspectProfile(ctx, *prompt.DefaultProfileID); err != nil {
findings = append(findings, fmt.Errorf("%s prompt %q default profile %q: %w", scope, id, *prompt.DefaultProfileID, err))
}
}
}
input, _ := catalog.Inputs.Spec(resolved.Input.Module)
check(pipeline.StageInput, "", "", resolved.Input.Module, "", resolved.InputExecutionClass, resolved.Input.LLMProfile, input.Prompts)
check(pipeline.StageChunk, "", "", resolved.Chunk.Module, "", resolved.ChunkExecutionClass, resolved.Chunk.LLMProfile, resolved.ChunkPrompts)
steps := make(map[string]string)
for _, step := range resolved.Steps {
for _, lane := range step.ArtifactLanes {
steps[lane.ID] = step.ID
check(pipeline.StageExtract, step.ID, lane.ID, lane.Extract.Module, "", lane.ExtractExecutionClass, lane.Extract.LLMProfile, lane.ExtractPrompts)
check(pipeline.StageMerge, step.ID, lane.ID, lane.Merge.Module, "", lane.MergeExecutionClass, lane.Merge.LLMProfile, lane.MergePrompts)
check(pipeline.StageNormalize, step.ID, lane.ID, lane.Normalize.Module, "", lane.NormalizeExecutionClass, lane.Normalize.LLMProfile, lane.NormalizePrompts)
}
}
for _, chain := range resolved.ValidatorChains {
for _, validator := range chain.Validators {
check(chain.Stage, steps[chain.LaneID], chain.LaneID, chain.ModuleKey, validator.Binding.Module, validator.ExecutionClass, validator.Binding.LLMProfile, validator.Prompts)
}
}
output, _ := catalog.Outputs.Spec(resolved.Output.Module)
check(pipeline.StageOutput, "", "", resolved.Output.Module, "", resolved.OutputExecutionClass, resolved.Output.LLMProfile, output.Prompts)
if err := ctx.Err(); err != nil {
findings = append(findings, err)
}
return errors.Join(findings...)
}

View File

@@ -0,0 +1,163 @@
package preflight
import (
"context"
"errors"
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
"go.maximumdirect.net/configkit"
"go.maximumdirect.net/promptkit"
)
type noGeneration struct{ t *testing.T }
func (client noGeneration) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
client.t.Fatal("offline preflight invoked a provider")
return nil, nil
}
func inspectionEngine(t *testing.T) *promptkit.Engine {
t.Helper()
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{promptkit.SettingsInput(configkit.Bytes("offline resources", []byte(`profiles:
- id: partial
model: fixture
api_key_env: NOTARIUS_PREFLIGHT_MUST_NOT_REQUIRE_THIS_CREDENTIAL
api_key_required: true
prompts:
- id: initial
default_profile: partial
inputs: [{name: source, required: true}]
messages: [{role: user, content: '{{input "source"}}'}]
output: {format: text, validation_mode: none}
- id: correction
default_profile: partial
messages: [{role: user, content: 'Correct the previous response.'}]
output: {format: text, validation_mode: none}
- id: validator
default_profile: partial
messages: [{role: user, content: 'Assess.'}]
output: {format: text, validation_mode: none}
- id: unknown-default
default_profile: missing-profile
messages: [{role: user, content: 'Select a profile.'}]
output: {format: text, validation_mode: none}
- id: no-default
messages: [{role: user, content: 'Select a profile.'}]
output: {format: text, validation_mode: none}
`)))}}}}}, promptkit.WithBundledOpenRouter(false), promptkit.WithLLMClient(noGeneration{t}))
if err != nil {
t.Fatal(err)
}
return engine
}
func resolvedFixture() pipeline.ResolvedPipeline {
return pipeline.ResolvedPipeline{ID: "campaign", Steps: []pipeline.ResolvedPipelineStep{{ID: "identify", ArtifactLanes: []pipeline.ResolvedArtifactLane{{ID: "characters", Extract: pipeline.Binding("character-producer"), ExtractExecutionClass: contracts.ExecutionClassLLMBacked, ExtractPrompts: []string{"initial", "correction"}}}}},
ValidatorChains: []pipeline.ResolvedValidatorChain{{Stage: pipeline.StageExtract, LaneID: "characters", ModuleKey: "character-producer", Validators: []pipeline.ResolvedValidator{{Binding: pipeline.Binding("domain-validator"), ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{"validator"}}}}},
}
}
func TestOfflinePreflightAcceptsPartialDefaultsWithoutExecutionInputs(t *testing.T) {
engine := inspectionEngine(t)
resolved := resolvedFixture()
resolved.Steps[0].ArtifactLanes[0].ExtractReferences.Bindings = []pipeline.ReferenceBinding{{Source: "/not-present/reference.json"}}
if !RequiresEngine(resolved) {
t.Fatal("selected LLM roles were missed")
}
if err := Validate(context.Background(), resolved, pipeline.ModuleCatalog{}, engine); err != nil {
t.Fatalf("partial profile or absent execution inputs rejected: %v", err)
}
}
func TestOfflinePreflightReportsEveryKnownResourceWithScopeAndCause(t *testing.T) {
resolved := resolvedFixture()
lane := &resolved.Steps[0].ArtifactLanes[0]
lane.Extract.LLMProfile = "missing-explicit"
lane.ExtractPrompts = []string{"missing-initial", "missing-correction"}
resolved.ValidatorChains[0].Validators[0].Prompts = []string{"missing-validator"}
err := Validate(context.Background(), resolved, pipeline.ModuleCatalog{}, inspectionEngine(t))
if !errors.Is(err, promptkit.ErrPromptNotFound) || !errors.Is(err, promptkit.ErrProfileNotFound) {
t.Fatalf("native causes lost: %v", err)
}
for _, value := range []string{"campaign", "identify", "characters", "extract", "character-producer", "domain-validator", "missing-explicit", "missing-initial", "missing-correction", "missing-validator"} {
if !strings.Contains(err.Error(), value) {
t.Fatalf("missing preflight context %q: %v", value, err)
}
}
}
func TestOfflinePreflightChecksApplicableProfilesButNotExecutionReadiness(t *testing.T) {
engine := inspectionEngine(t)
for _, test := range []struct {
prompt, profile string
want error
}{
{"unknown-default", "", promptkit.ErrProfileNotFound},
{"no-default", "", nil},
{"unknown-default", "partial", nil},
{"no-default", "partial", nil},
} {
resolved := resolvedFixture()
lane := &resolved.Steps[0].ArtifactLanes[0]
lane.ExtractPrompts, lane.Extract.LLMProfile = []string{test.prompt}, test.profile
err := Validate(context.Background(), resolved, pipeline.ModuleCatalog{}, engine)
if test.want == nil && err != nil || test.want != nil && !errors.Is(err, test.want) {
t.Fatalf("prompt=%q profile=%q: %v", test.prompt, test.profile, err)
}
}
}
func TestOfflinePreflightLeavesDynamicSelectionsForRuntime(t *testing.T) {
resolved := resolvedFixture()
resolved.Steps[0].ArtifactLanes[0].ExtractPrompts = nil
resolved.ValidatorChains = nil
if err := Validate(context.Background(), resolved, pipeline.ModuleCatalog{}, inspectionEngine(t)); err != nil {
t.Fatal(err)
}
}
func TestOfflinePreflightIncludesInputOutputAndNestedValidatorsWithoutConstruction(t *testing.T) {
catalog := pipeline.ModuleCatalog{Inputs: pipeline.NewInputAdapterRegistry(), Outputs: pipeline.NewOutputEncoderRegistry()}
for _, stage := range []pipeline.ModuleStage{pipeline.StageInput, pipeline.StageOutput} {
spec := pipeline.ModuleSpec{Key: string(stage), Stage: stage, ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{"missing-" + string(stage)}}
var err error
if stage == pipeline.StageInput {
err = catalog.Inputs.RegisterWithSpec(spec, func() (contracts.InputAdapter, error) { t.Fatal("preflight constructed input"); return nil, nil })
} else {
err = catalog.Outputs.RegisterWithSpec(spec, func() (contracts.OutputEncoder, error) { t.Fatal("preflight constructed output"); return nil, nil })
}
if err != nil {
t.Fatal(err)
}
}
resolved := pipeline.ResolvedPipeline{ID: "io", Input: pipeline.Binding("input"), InputExecutionClass: contracts.ExecutionClassLLMBacked, Output: pipeline.Binding("output"), OutputExecutionClass: contracts.ExecutionClassLLMBacked}
err := Validate(context.Background(), resolved, catalog, inspectionEngine(t))
if !errors.Is(err, promptkit.ErrPromptNotFound) || !strings.Contains(err.Error(), "missing-input") || !strings.Contains(err.Error(), "missing-output") {
t.Fatalf("input/output declarations were missed: %v", err)
}
resolved = resolvedFixture()
resolved.Steps[0].ArtifactLanes[0].ExtractExecutionClass = contracts.ExecutionClassDeterministic
if !RequiresEngine(resolved) {
t.Fatal("nested validator omitted from engine selection")
}
}
func TestDeterministicPreflightNeedsNoEngineAndCancellationStopsInspection(t *testing.T) {
if RequiresEngine(pipeline.ResolvedPipeline{}) {
t.Fatal("deterministic composition requires engine")
}
if err := Validate(context.Background(), pipeline.ResolvedPipeline{}, pipeline.ModuleCatalog{}, nil); err != nil {
t.Fatal(err)
}
if err := Validate(context.Background(), resolvedFixture(), pipeline.ModuleCatalog{}, nil); !errors.Is(err, promptkit.ErrInvalidConfig) {
t.Fatalf("missing engine: %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if err := Validate(ctx, resolvedFixture(), pipeline.ModuleCatalog{}, nil); !errors.Is(err, context.Canceled) {
t.Fatalf("cancellation lost: %v", err)
}
}