From 2a9c5a60757c4a97d7b01e6a2b7bacf49d382660 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sun, 22 Nov 2026 03:32:31 +0000 Subject: [PATCH] Add offline pipeline inspection and shared native resource preflight --- docs/roadmap/implementation.md | 21 ++ internal/cli/application.go | 11 +- internal/cli/application_test.go | 3 +- internal/cli/configkit_load.go | 7 + internal/cli/pipeline_commands.go | 143 +++++++++++++ internal/cli/pipeline_commands_test.go | 276 +++++++++++++++++++++++++ internal/cli/pipeline_description.go | 72 +++++++ internal/preflight/preflight.go | 120 +++++++++++ internal/preflight/preflight_test.go | 163 +++++++++++++++ 9 files changed, 806 insertions(+), 10 deletions(-) create mode 100644 internal/cli/pipeline_commands.go create mode 100644 internal/cli/pipeline_commands_test.go create mode 100644 internal/cli/pipeline_description.go create mode 100644 internal/preflight/preflight.go create mode 100644 internal/preflight/preflight_test.go diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 5ccc2e13..81672c71 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -1503,6 +1503,27 @@ operations. Check meaningful evidence, not the complete native JSON shape. ## Stage 32: Add Offline Preflight And Pipeline Inspection Commands +**Status:** Complete. Added native pipeline list/show/validate declarations using +the shared captured application load. List enumerates sorted configuration IDs +without constructing a catalog or engine. Show uses ordinary domain resolution +and reports steps, lanes, modules, controls, validator chains, prompt declarations, +and reference bindings without expanding profiles or serializing runtime options. +Validation resolves the same domain contracts and supplies the captured engine +only for selected model work. The new preflight package inspects every known +initial/correction/validator/input/output prompt and explicit/applicable default +profile, preserves native causes, and aggregates pipeline/step/lane/validator +context. Partial profiles pass; undeclared dynamic selections remain runtime +checks. No request preparation, transcript/reference reads, credential lookup, +provider call, module construction, or run-storage creation is involved. +Focused real-kit preflight and CLI command tests, vet, repeated race checks, +focused pipeline tests, production preflight/pipeline/config builds, and diff +checks pass. Native configuration validation retains no pipeline-specific mode; +added assembled help/config-command regressions. The full CLI suite, including +those assembled regressions, remains blocked at setup by removed reporting-package +imports and old ConfigKit's YAML plugin/limit dependency; it is not claimed passing. +Scheduled cleanup and final verification must execute it. Native run-handler +composition will reuse this preflight boundary in Stage 33. + **Goal:** execution and inspection share domain resolution and known dependencies. **Inspect/change:** new `internal/preflight`, CLI pipeline declarations, catalog, diff --git a/internal/cli/application.go b/internal/cli/application.go index 1590260b..0d0e1874 100644 --- a/internal/cli/application.go +++ b/internal/cli/application.go @@ -2,10 +2,9 @@ package cli import ( "context" - "fmt" - "strings" "gitea.maximumdirect.net/eric/notarius/internal/core/config" + "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "go.maximumdirect.net/ckconfig" "go.maximumdirect.net/ckprompt" "go.maximumdirect.net/commandkit" @@ -42,6 +41,7 @@ func NewApplication(version string, options Options) (*commandkit.App, error) { definitions = ckprompt.WithNamespaces(definitions, promptkit.Namespaces{}) declarations := []commandkit.Declaration{ commandkit.Completion("notarius"), + pipelineCommands(newState, func() (pipeline.ModuleCatalog, error) { return effectiveCatalog(options) }), ckconfig.Command(ckconfig.Suite[struct{}, *nativeLoaders]{ NewState: newState, CheckInput: func(_ string, selection ckconfig.Selection, _ struct{}) error { @@ -53,10 +53,3 @@ func NewApplication(version string, options Options) (*commandkit.App, error) { declarations = append(declarations, ckprompt.Declarations(definitions)...) return commandkit.New(commandkit.Identity{Name: "notarius", Version: version}, declarations...) } - -func checkConfigSelection(selection ckconfig.Selection) error { - if selection.Config.Set && strings.TrimSpace(selection.Config.Value) == "" { - return fmt.Errorf("--config must not be empty") - } - return nil -} diff --git a/internal/cli/application_test.go b/internal/cli/application_test.go index 4c40a408..403fe8d7 100644 --- a/internal/cli/application_test.go +++ b/internal/cli/application_test.go @@ -17,7 +17,7 @@ func TestNativeApplicationBuiltinsNeedNoConfiguration(t *testing.T) { if err != nil { t.Fatal(err) } - for _, args := range [][]string{{"--help"}, {"help"}, {"version"}, {"completion", "bash"}, {"completion", "zsh"}, {"completion", "fish"}, {"config", "--help"}} { + for _, args := range [][]string{{"--help"}, {"help"}, {"version"}, {"completion", "bash"}, {"completion", "zsh"}, {"completion", "fish"}, {"config", "--help"}, {"pipeline", "--help"}, {"pipeline", "validate", "--help"}} { var out, diagnostics bytes.Buffer outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &out, Err: &diagnostics}) if outcome.Status != 0 || out.Len() == 0 { @@ -52,6 +52,7 @@ func TestNativeConfigurationScopesAndDiffUseSelectedLoadEvidence(t *testing.T) { {"differences", []string{"config", "diff", "--config", left, "--against-config", right, "--check"}, 3, "workers"}, {"identical", []string{"config", "diff", "--config", left, "--against-config", left, "--check"}, 0, ""}, {"empty", []string{"config", "show", "--config", ""}, 2, ""}, + {"configuration has no pipeline mode", []string{"config", "validate", "--config", left, "--pipeline", "demo"}, 2, ""}, } { t.Run(test.name, func(t *testing.T) { var out, diagnostics bytes.Buffer diff --git a/internal/cli/configkit_load.go b/internal/cli/configkit_load.go index d2c44f53..c13749d8 100644 --- a/internal/cli/configkit_load.go +++ b/internal/cli/configkit_load.go @@ -31,6 +31,13 @@ type loadedSelection struct { engine *promptkit.Engine } +func checkConfigSelection(selection ckconfig.Selection) error { + if selection.Config.Set && strings.TrimSpace(selection.Config.Value) == "" { + return fmt.Errorf("--config must not be empty") + } + return nil +} + func (loaders *nativeLoaders) application(ctx context.Context, selection ckconfig.Selection) (*configkit.Result[config.Document], error) { if err := ctx.Err(); err != nil { return nil, err diff --git a/internal/cli/pipeline_commands.go b/internal/cli/pipeline_commands.go new file mode 100644 index 00000000..6dceef90 --- /dev/null +++ b/internal/cli/pipeline_commands.go @@ -0,0 +1,143 @@ +package cli + +import ( + "context" + "errors" + "fmt" + "sort" + "strings" + "text/template" + + "gitea.maximumdirect.net/eric/notarius/internal/core/config" + "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" + "gitea.maximumdirect.net/eric/notarius/internal/preflight" + "go.maximumdirect.net/ckconfig" + "go.maximumdirect.net/commandkit" + "go.maximumdirect.net/configkit" + "go.maximumdirect.net/promptkit" +) + +type pipelineInspectionInput struct { + Selection ckconfig.Selection + PipelineID string +} + +func configurationSelectionFlags[I any](destination func(*I) *ckconfig.Selection) []commandkit.Flag[I] { + return []commandkit.Flag[I]{ + commandkit.ValueFlag(commandkit.FlagMeta{Name: "config", Usage: "Select the application configuration.", Completion: commandkit.CompleteFiles()}, (*string)(nil), func(value string) (string, error) { + if strings.TrimSpace(value) == "" { + return "", fmt.Errorf("--config must not be empty") + } + return value, nil + }, func(input *I, value commandkit.Supplied[string]) { destination(input).Config = value }), + commandkit.ValueFlag(commandkit.FlagMeta{Name: "overlay", Usage: "Select a configuration overlay.", Completion: commandkit.CompleteFiles()}, (*string)(nil), func(value string) (string, error) { return value, nil }, func(input *I, value commandkit.Supplied[string]) { destination(input).Overlay = value }), + commandkit.RepeatFlag(commandkit.FlagMeta{Name: "set", Usage: "Override POINTER=JSON."}, []configkit.Override(nil), parseConfigurationOverride, func(input *I, value commandkit.Supplied[[]configkit.Override]) { + destination(input).Overrides = value.Value + }), + } +} + +func parseConfigurationOverride(raw string) (configkit.Override, error) { + pointer, encoded, ok := strings.Cut(raw, "=") + if !ok { + return configkit.Override{}, fmt.Errorf("override requires POINTER=JSON") + } + path, err := configkit.ParsePath(pointer) + if err != nil { + return configkit.Override{}, err + } + value, err := commandkit.ParseCLIJSON(encoded) + if err != nil { + return configkit.Override{}, err + } + return configkit.Override{Path: path, Value: value, Flag: "--set"}, nil +} + +func pipelineCommands(newState func() *nativeLoaders, catalog func() (pipeline.ModuleCatalog, error)) commandkit.Command { + command := func(operation, summary string) commandkit.Command { + var arguments []commandkit.Argument[pipelineInspectionInput] + if operation != "list" { + arguments = []commandkit.Argument[pipelineInspectionInput]{commandkit.ValueArgument(commandkit.ArgumentMeta{Name: "pipeline-id", Usage: "Configured pipeline identifier."}, true, func(value string) (string, error) { + value = strings.TrimSpace(value) + if value == "" { + return "", fmt.Errorf("pipeline id must not be empty") + } + return value, nil + }, func(input *pipelineInspectionInput, value commandkit.Supplied[string]) { + input.PipelineID = value.Value + })} + } + return commandkit.NewCommand(commandkit.CommandSpec[pipelineInspectionInput]{Name: operation, Summary: summary, + Flags: configurationSelectionFlags(func(input *pipelineInspectionInput) *ckconfig.Selection { return &input.Selection }), Arguments: arguments, + CheckInput: func(input pipelineInspectionInput) error { return checkConfigSelection(input.Selection) }, + Run: func(invocation commandkit.Invocation, input pipelineInspectionInput) commandkit.Outcome { + state := newState() + result, err := state.application(invocation.Context, input.Selection) + if err != nil { + return pipelineConfigurationFailure(err) + } + if operation == "list" { + ids := make([]string, 0, len(result.Config.Pipelines)) + for id := range result.Config.Pipelines { + ids = append(ids, id) + } + sort.Strings(ids) + return reportPipelineInspection(invocation, commandkit.Report{Value: ids}) + } + loaded, err := config.FromResult(result) + if err != nil { + return pipelineInvalidConfiguration(err) + } + modules, err := catalog() + if err != nil { + return commandkit.Failure(err.Error()).WithCause(err) + } + effective, err := loaded.RuntimeConfig().Resolve(config.ResolveInput{PipelineID: input.PipelineID, Catalog: modules}) + if err != nil { + return pipelineInvalidConfiguration(err) + } + if operation == "show" { + return reportPipelineInspection(invocation, commandkit.Report{Value: describePipeline(effective.ResolvedPipeline, modules)}) + } + var engine *promptkit.Engine + if preflight.RequiresEngine(effective.ResolvedPipeline) { + engine, err = state.engine(invocation.Context, input.Selection) + if err != nil { + return pipelineConfigurationFailure(fmt.Errorf("pipeline %q native resources: %w", input.PipelineID, err)) + } + } + if err := preflight.Validate(invocation.Context, effective.ResolvedPipeline, modules, engine); err != nil { + return pipelineInvalidConfiguration(err) + } + return reportPipelineInspection(invocation, commandkit.Report{Value: struct { + Pipeline string `json:"pipeline"` + Valid bool `json:"valid"` + }{input.PipelineID, true}, TextTemplate: pipelineValidationTemplate}) + }, + }) + } + return commandkit.Group("pipeline", "Inspect, validate, or execute configured pipelines.", command("list", "List configured pipeline identifiers."), command("show", "Show a resolved pipeline composition."), command("validate", "Validate known pipeline dependencies offline.")) +} + +var pipelineValidationTemplate = template.Must(commandkit.ParseTextTemplate("pipeline-validation", "Pipeline {{printf \"%q\" .Pipeline}} is valid.\n", nil)) + +func reportPipelineInspection(invocation commandkit.Invocation, report commandkit.Report) commandkit.Outcome { + if err := invocation.Reporter().Report(report); err != nil { + return commandkit.Failure("Could not report pipeline inspection.").WithCause(err) + } + return commandkit.Success() +} + +func pipelineConfigurationFailure(err error) commandkit.Outcome { + if ckconfig.ClassifyConfigurationError(err) == commandkit.OutcomeUsage { + return commandkit.Usage(err.Error()).WithCause(err) + } + return commandkit.Failure(err.Error()).WithCause(err) +} + +func pipelineInvalidConfiguration(err error) commandkit.Outcome { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return commandkit.Failure(err.Error()).WithCause(err) + } + return commandkit.Usage(err.Error()).WithCause(err) +} diff --git a/internal/cli/pipeline_commands_test.go b/internal/cli/pipeline_commands_test.go new file mode 100644 index 00000000..eba10936 --- /dev/null +++ b/internal/cli/pipeline_commands_test.go @@ -0,0 +1,276 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" + "go.maximumdirect.net/commandkit" + "go.maximumdirect.net/promptkit" +) + +const inspectionConfiguration = `version: 6 +output: {directory: output-must-not-exist} +cache_directory: cache-must-not-exist +debug: {directory: debug-must-not-exist} +pipelines: + demo: + input: {module: fixture/input} + chunk: {module: fixture/chunk} + output: {module: fixture/output} + steps: + - id: build + artifacts: + notes: + extract: + module: fixture/extract + references: {context: {path: reference-must-not-exist.txt}} + validators: [{module: fixture/validator}] + merge: {module: fixture/merge} + normalize: {module: fixture/normalize} +promptkit: + profiles: + - id: partial + model: fixture-model + api_key_env: NOTARIUS_INSPECTION_NO_CREDENTIAL_REQUIRED + api_key_required: true + prompts: + - id: inspect-initial + default_profile: partial + inputs: [{name: source, required: true}] + messages: [{role: user, content: '{{input "source"}}'}] + output: {format: text, validation_mode: none} + - id: inspect-correction + default_profile: partial + messages: [{role: user, content: Correct.}] + output: {format: text, validation_mode: none} + - id: inspect-validator + default_profile: partial + messages: [{role: user, content: Assess.}] + output: {format: text, validation_mode: none} +` + +type inspectionArtifact struct { + Value string `json:"value"` +} +type inspectionCodec struct{} + +func (inspectionCodec) Kind() contracts.ArtifactKind { return "test/inspection" } +func (inspectionCodec) MediaType() string { return "application/json" } +func (inspectionCodec) Schema() contracts.ArtifactSchema { + return contracts.ArtifactSchema{ID: "test.inspection", Name: "inspection", Version: "v1", JSONSchema: []byte(`{"type":"object","properties":{"value":{"type":"string"}},"required":["value"]}`)} +} +func (inspectionCodec) Encode(value inspectionArtifact) ([]byte, error) { return json.Marshal(value) } +func (codec inspectionCodec) EncodeCandidate(value inspectionArtifact) ([]byte, error) { + return codec.Encode(value) +} +func (codec inspectionCodec) DecodeCandidate(content []byte) (inspectionArtifact, error) { + return codec.Decode(content) +} +func (inspectionCodec) Decode(content []byte) (inspectionArtifact, error) { + var value inspectionArtifact + err := json.Unmarshal(content, &value) + return value, err +} + +func inspectionCatalog(t *testing.T, class contracts.ExecutionClass) pipeline.ModuleCatalog { + t.Helper() + catalog := pipeline.ModuleCatalog{Inputs: pipeline.NewInputAdapterRegistry(), Chunkers: pipeline.NewChunkerRegistry(), ArtifactCodecs: pipeline.NewArtifactCodecRegistry(), Extractors: pipeline.NewExtractorRegistry(), Mergers: pipeline.NewMergerRegistry(), Normalizers: pipeline.NewNormalizerRegistry(), Validators: pipeline.NewValidatorRegistry(), ValidatorChains: pipeline.NewValidatorChainRegistry(), Outputs: pipeline.NewOutputEncoderRegistry()} + check := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + spec := func(stage pipeline.ModuleStage, key string) pipeline.ModuleSpec { + return pipeline.ModuleSpec{Key: key, Stage: stage, ExecutionClass: contracts.ExecutionClassDeterministic} + } + check(catalog.Inputs.RegisterWithSpec(spec(pipeline.StageInput, "fixture/input"), func() (contracts.InputAdapter, error) { t.Fatal("inspection constructed input"); return nil, nil })) + check(catalog.Chunkers.RegisterWithSpec(spec(pipeline.StageChunk, "fixture/chunk"), func() (contracts.Chunker, error) { t.Fatal("inspection constructed chunker"); return nil, nil })) + check(catalog.Outputs.RegisterWithSpec(spec(pipeline.StageOutput, "fixture/output"), func() (contracts.OutputEncoder, error) { t.Fatal("inspection constructed output"); return nil, nil })) + check(pipeline.RegisterArtifactCodec(catalog.ArtifactCodecs, inspectionCodec{})) + extract := spec(pipeline.StageExtract, "fixture/extract") + extract.ArtifactKind, extract.ExecutionClass, extract.Prompts = "test/inspection", class, []string{"inspect-initial", "inspect-correction"} + extract.ReferenceSlots = []contracts.ReferenceSlot{{Name: "context", Required: true, AcceptedMediaTypes: []string{"text/plain"}}} + check(pipeline.RegisterExtractor(catalog.Extractors, extract, func() (contracts.Extractor[inspectionArtifact], error) { + t.Fatal("inspection constructed extractor") + return nil, nil + })) + merge := spec(pipeline.StageMerge, "fixture/merge") + merge.ArtifactKind = "test/inspection" + check(pipeline.RegisterMerger(catalog.Mergers, merge, func() (contracts.Merger[inspectionArtifact], error) { + t.Fatal("inspection constructed merger") + return nil, nil + })) + normalize := spec(pipeline.StageNormalize, "fixture/normalize") + normalize.ArtifactKind = "test/inspection" + check(pipeline.RegisterNormalizer(catalog.Normalizers, normalize, func() (contracts.Normalizer[inspectionArtifact], error) { + t.Fatal("inspection constructed normalizer") + return nil, nil + })) + check(pipeline.RegisterTypedValidator(catalog.Validators, "test/inspection", pipeline.ValidatorSpec{Key: "fixture/validator", ExecutionClass: class, Prompts: []string{"inspect-validator"}}, func() (contracts.TypedValidator[inspectionArtifact], error) { + t.Fatal("inspection constructed validator") + return nil, nil + })) + return catalog +} + +type inspectionProvider struct{ t *testing.T } + +func (provider inspectionProvider) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + provider.t.Fatal("inspection called provider") + return nil, nil +} + +func writeInspectionConfiguration(t *testing.T, content string) string { + t.Helper() + file := filepath.Join(t.TempDir(), "config.yml") + if err := os.WriteFile(file, []byte(content), 0600); err != nil { + t.Fatal(err) + } + return file +} + +func newInspectionApplication(t *testing.T, catalog func() (pipeline.ModuleCatalog, error)) (*commandkit.App, **nativeLoaders) { + t.Helper() + var last *nativeLoaders + app, err := commandkit.New(commandkit.Identity{Name: "notarius"}, pipelineCommands(func() *nativeLoaders { + last = &nativeLoaders{engineOptions: []promptkit.Option{promptkit.WithLLMClient(inspectionProvider{t})}} + return last + }, catalog)) + if err != nil { + t.Fatal(err) + } + return app, &last +} + +func TestPipelineListOnlyEnumeratesCapturedConfiguration(t *testing.T) { + content := strings.Replace(inspectionConfiguration, " demo:", " zeta:", 1) + content = strings.Replace(content, "promptkit:", " alpha:\n input: {module: unavailable}\n artifacts:\n value:\n extract: {module: unavailable}\npromptkit:", 1) + file := writeInspectionConfiguration(t, content) + app, state := newInspectionApplication(t, func() (pipeline.ModuleCatalog, error) { + t.Fatal("list constructed a catalog") + return pipeline.ModuleCatalog{}, nil + }) + var out, diagnostics bytes.Buffer + outcome := app.Execute(context.Background(), []string{"pipeline", "list", "--config", file, "--format", "json"}, commandkit.Streams{Out: &out, Err: &diagnostics}) + var ids []string + if outcome.Status != 0 || json.Unmarshal(out.Bytes(), &ids) != nil || !reflect.DeepEqual(ids, []string{"alpha", "zeta"}) { + t.Fatalf("list=%#v %s %s", outcome, out.String(), diagnostics.String()) + } + if len((*state).selections) != 1 || (*state).selections[0].engine != nil { + t.Fatal("list expanded native resources") + } +} + +func TestPipelineShowResolvesCompositionWithoutEngineOrExecutionInputs(t *testing.T) { + file := writeInspectionConfiguration(t, inspectionConfiguration) + catalog := inspectionCatalog(t, contracts.ExecutionClassLLMBacked) + app, state := newInspectionApplication(t, func() (pipeline.ModuleCatalog, error) { return catalog, nil }) + var out, diagnostics bytes.Buffer + outcome := app.Execute(context.Background(), []string{"pipeline", "show", "demo", "--config", file, "--format", "json"}, commandkit.Streams{Out: &out, Err: &diagnostics}) + var report pipelineDescription + if err := json.Unmarshal(out.Bytes(), &report); err != nil || outcome.Status != 0 { + t.Fatalf("show=%#v %s %s %v", outcome, out.String(), diagnostics.String(), err) + } + if report.Pipeline != "demo" || len(report.Steps) != 1 || report.Steps[0].ID != "build" || len(report.Steps[0].Lanes) != 1 || report.Steps[0].Lanes[0].Extract.Module != "fixture/extract" || len(report.Steps[0].Lanes[0].Extract.References) != 1 || len(report.Validators) == 0 { + t.Fatalf("missing resolved composition: %#v", report) + } + if (*state).selections[0].engine != nil { + t.Fatal("show expanded profiles by constructing engine") + } + assertInspectionNoRunStorage(t, file) +} + +func TestPipelineValidateChecksNativeDeclarationsOfflineAndUsesOneLoad(t *testing.T) { + for _, class := range []contracts.ExecutionClass{contracts.ExecutionClassDeterministic, contracts.ExecutionClassLLMBacked} { + file := writeInspectionConfiguration(t, inspectionConfiguration) + catalog := inspectionCatalog(t, class) + app, state := newInspectionApplication(t, func() (pipeline.ModuleCatalog, error) { return catalog, nil }) + var out, diagnostics bytes.Buffer + outcome := app.Execute(context.Background(), []string{"pipeline", "validate", "demo", "--config", file}, commandkit.Streams{Out: &out, Err: &diagnostics}) + if outcome.Status != 0 || !strings.Contains(out.String(), "demo") { + t.Fatalf("validate=%#v %s %s", outcome, out.String(), diagnostics.String()) + } + if len((*state).selections) != 1 || ((*state).selections[0].engine != nil) != (class == contracts.ExecutionClassLLMBacked) { + t.Fatal("validation loaded twice or built an unnecessary engine") + } + assertInspectionNoRunStorage(t, file) + } +} + +func TestPipelineValidateRejectsKnownMissingResourcesWithScope(t *testing.T) { + for _, id := range []string{"inspect-initial", "inspect-correction", "inspect-validator"} { + file := writeInspectionConfiguration(t, strings.Replace(inspectionConfiguration, "id: "+id, "id: unused-"+id, 1)) + catalog := inspectionCatalog(t, contracts.ExecutionClassLLMBacked) + app, _ := newInspectionApplication(t, func() (pipeline.ModuleCatalog, error) { return catalog, nil }) + var out, diagnostics bytes.Buffer + outcome := app.Execute(context.Background(), []string{"pipeline", "validate", "demo", "--config", file}, commandkit.Streams{Out: &out, Err: &diagnostics}) + if outcome.Status != 2 || !errors.Is(outcome.Cause, promptkit.ErrPromptNotFound) || !strings.Contains(diagnostics.String(), id) || !strings.Contains(diagnostics.String(), "build") || !strings.Contains(diagnostics.String(), "notes") { + t.Fatalf("known dependency=%#v %s", outcome, diagnostics.String()) + } + } +} + +func TestPipelineInspectionUsesNativeArgumentsSelectionsAndDelivery(t *testing.T) { + file := writeInspectionConfiguration(t, inspectionConfiguration) + catalog := inspectionCatalog(t, contracts.ExecutionClassLLMBacked) + app, _ := newInspectionApplication(t, func() (pipeline.ModuleCatalog, error) { return catalog, nil }) + for _, test := range []struct { + args []string + status int + }{ + {[]string{"pipeline", "show"}, 2}, + {[]string{"pipeline", "show", "demo", "--config", "", "--config", file}, 2}, + {[]string{"pipeline", "show", "unknown", "--config", file}, 2}, + {[]string{"pipeline", "show", "demo", "--config", file, "--set", `/pipelines/demo/steps/0/artifacts/notes/extract/options={"unsupported":true}`}, 2}, + {[]string{"pipeline", "validate", "demo", "--config", file, "--set", `/pipelines/demo/llm_profile="missing-profile"`}, 2}, + {[]string{"pipeline", "show", "demo", "--config", file, "--set", `/pipelines/demo/steps/0/artifacts/notes/extract/references=null`}, 2}, + {[]string{"pipeline", "list", "--config", file, "--set", `/pipelines/demo/input/module="another-input"`}, 0}, + {[]string{"pipeline", "validate", "demo", "--config", file, "--quiet"}, 0}, + {[]string{"pipeline", "--help"}, 0}, + } { + var out, diagnostics bytes.Buffer + outcome := app.Execute(context.Background(), test.args, commandkit.Streams{Out: &out, Err: &diagnostics}) + if outcome.Status != test.status { + t.Fatalf("%v: %#v %s", test.args, outcome, diagnostics.String()) + } + if test.args[len(test.args)-1] == "--quiet" && out.Len() != 0 { + t.Fatal("quiet pipeline inspection wrote stdout") + } + } +} + +type inspectionFailWriter struct{ err error } + +func (writer inspectionFailWriter) Write([]byte) (int, error) { return 0, writer.err } + +func TestPipelineInspectionReportFailureRemainsOperational(t *testing.T) { + file := writeInspectionConfiguration(t, inspectionConfiguration) + app, _ := newInspectionApplication(t, func() (pipeline.ModuleCatalog, error) { + t.Fatal("list loaded catalog") + return pipeline.ModuleCatalog{}, nil + }) + primary := errors.New("inspection stdout delivery failed") + var diagnostics bytes.Buffer + outcome := app.Execute(context.Background(), []string{"pipeline", "list", "--config", file}, commandkit.Streams{Out: inspectionFailWriter{primary}, Err: &diagnostics}) + if outcome.Status != 1 || !errors.Is(outcome.Cause, primary) { + t.Fatalf("delivery failure classified as invalid configuration: %#v %s", outcome, diagnostics.String()) + } +} + +func assertInspectionNoRunStorage(t *testing.T, file string) { + t.Helper() + entries, err := os.ReadDir(filepath.Dir(file)) + if err != nil || len(entries) != 1 || entries[0].Name() != "config.yml" { + t.Fatalf("inspection read/materialized inputs or created storage: %v %v", entries, err) + } +} diff --git a/internal/cli/pipeline_description.go b/internal/cli/pipeline_description.go new file mode 100644 index 00000000..19eb9cd0 --- /dev/null +++ b/internal/cli/pipeline_description.go @@ -0,0 +1,72 @@ +package cli + +import ( + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" +) + +// Inspection reports identities and resolved composition, not another effective +// configuration or profile expansion. ConfigKit owns secret-aware settings views. +type pipelineModuleDescription struct { + Module string `json:"module"` + ExecutionClass contracts.ExecutionClass `json:"execution_class"` + LLMProfile string `json:"llm_profile,omitempty"` + Controls *contracts.ProducerControls `json:"controls,omitempty"` + Prompts []string `json:"prompts,omitempty"` + References []pipeline.ReferenceBinding `json:"references,omitempty"` +} + +type pipelineLaneDescription struct { + ID string `json:"id"` + ArtifactKind contracts.ArtifactKind `json:"artifact_kind"` + Extract pipelineModuleDescription `json:"extract"` + Merge pipelineModuleDescription `json:"merge"` + Normalize pipelineModuleDescription `json:"normalize"` +} + +type pipelineStepDescription struct { + ID string `json:"id"` + Lanes []pipelineLaneDescription `json:"lanes"` +} +type pipelineValidatorDescription struct { + Stage pipeline.ModuleStage `json:"stage"` + Lane string `json:"lane,omitempty"` + Module string `json:"module"` + Validators []pipelineModuleDescription `json:"validators"` +} +type pipelineDescription struct { + Pipeline string `json:"pipeline"` + Input pipelineModuleDescription `json:"input"` + Chunk pipelineModuleDescription `json:"chunk"` + Steps []pipelineStepDescription `json:"steps"` + Validators []pipelineValidatorDescription `json:"validators,omitempty"` + Output pipelineModuleDescription `json:"output"` +} + +func describePipeline(resolved pipeline.ResolvedPipeline, catalog pipeline.ModuleCatalog) pipelineDescription { + module := func(binding pipeline.ModuleBinding, class contracts.ExecutionClass, prompts []string, controls *contracts.ProducerControls, references []pipeline.ReferenceBinding) pipelineModuleDescription { + return pipelineModuleDescription{Module: binding.Module, ExecutionClass: class, LLMProfile: binding.LLMProfile, Prompts: prompts, Controls: controls, References: references} + } + input, _ := catalog.Inputs.Spec(resolved.Input.Module) + output, _ := catalog.Outputs.Spec(resolved.Output.Module) + report := pipelineDescription{Pipeline: resolved.ID, Input: module(resolved.Input, resolved.InputExecutionClass, input.Prompts, nil, nil), + Chunk: module(resolved.Chunk, resolved.ChunkExecutionClass, resolved.ChunkPrompts, &resolved.ChunkControls, resolved.ChunkReferences.Bindings), Output: module(resolved.Output, resolved.OutputExecutionClass, output.Prompts, nil, nil)} + for _, step := range resolved.Steps { + description := pipelineStepDescription{ID: step.ID, Lanes: make([]pipelineLaneDescription, 0, len(step.ArtifactLanes))} + for _, lane := range step.ArtifactLanes { + description.Lanes = append(description.Lanes, pipelineLaneDescription{ID: lane.ID, ArtifactKind: lane.ArtifactKind, + Extract: module(lane.Extract, lane.ExtractExecutionClass, lane.ExtractPrompts, &lane.ExtractControls, lane.ExtractReferences.Bindings), + Merge: module(lane.Merge, lane.MergeExecutionClass, lane.MergePrompts, &lane.MergeControls, lane.MergeReferences.Bindings), + Normalize: module(lane.Normalize, lane.NormalizeExecutionClass, lane.NormalizePrompts, &lane.NormalizeControls, lane.NormalizeReferences.Bindings)}) + } + report.Steps = append(report.Steps, description) + } + for _, chain := range resolved.ValidatorChains { + description := pipelineValidatorDescription{Stage: chain.Stage, Lane: chain.LaneID, Module: chain.ModuleKey, Validators: make([]pipelineModuleDescription, 0, len(chain.Validators))} + for _, validator := range chain.Validators { + description.Validators = append(description.Validators, module(validator.Binding, validator.ExecutionClass, validator.Prompts, nil, nil)) + } + report.Validators = append(report.Validators, description) + } + return report +} diff --git a/internal/preflight/preflight.go b/internal/preflight/preflight.go new file mode 100644 index 00000000..64464ce0 --- /dev/null +++ b/internal/preflight/preflight.go @@ -0,0 +1,120 @@ +// Package preflight checks statically known native resources for an already +// resolved pipeline. Acquisition, construction, and presentation belong to callers. +package preflight + +import ( + "context" + "errors" + "fmt" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" + "go.maximumdirect.net/promptkit" +) + +// RequiresEngine reports selected model work, including nested validators. +// Deterministic pipelines need no engine merely to validate their composition. +func RequiresEngine(resolved pipeline.ResolvedPipeline) bool { + if resolved.InputExecutionClass == contracts.ExecutionClassLLMBacked || resolved.ChunkExecutionClass == contracts.ExecutionClassLLMBacked || resolved.OutputExecutionClass == contracts.ExecutionClassLLMBacked { + return true + } + for _, step := range resolved.Steps { + for _, lane := range step.ArtifactLanes { + if lane.ExtractExecutionClass == contracts.ExecutionClassLLMBacked || lane.MergeExecutionClass == contracts.ExecutionClassLLMBacked || lane.NormalizeExecutionClass == contracts.ExecutionClassLLMBacked { + return true + } + } + } + for _, chain := range resolved.ValidatorChains { + for _, validator := range chain.Validators { + if validator.ExecutionClass == contracts.ExecutionClassLLMBacked { + return true + } + } + } + return false +} + +// Validate checks the same declarations and resource selections used by native +// request builders. It does not render, prepare, read input/reference files, +// resolve credential values, or call providers. Dynamic selections remain runtime +// checks. Callers first use their ordinary application resolver for module, +// options, type, validator-chain, and ordered-reference compatibility. +func Validate(ctx context.Context, resolved pipeline.ResolvedPipeline, catalog pipeline.ModuleCatalog, engine *promptkit.Engine) error { + if ctx == nil { + return errors.New("preflight context must not be nil") + } + if err := ctx.Err(); err != nil { + return err + } + var findings []error + check := func(stage pipeline.ModuleStage, step, lane, module, validator string, class contracts.ExecutionClass, profile string, prompts []string) { + if class != contracts.ExecutionClassLLMBacked || ctx.Err() != nil { + return + } + scope := fmt.Sprintf("pipeline %q", resolved.ID) + if step != "" { + scope += fmt.Sprintf(" step %q", step) + } + if lane != "" { + scope += fmt.Sprintf(" lane %q", lane) + } + scope += fmt.Sprintf(" %s module %q", stage, module) + if validator != "" { + scope += fmt.Sprintf(" validator %q", validator) + } + if engine == nil { + findings = append(findings, fmt.Errorf("%s: %w: native engine is required", scope, promptkit.ErrInvalidConfig)) + return + } + if profile != "" { + if _, err := engine.InspectProfile(ctx, profile); err != nil { + findings = append(findings, fmt.Errorf("%s profile %q: %w", scope, profile, err)) + } + } + for _, id := range prompts { + if ctx.Err() != nil { + return + } + prompt, err := engine.InspectPrompt(ctx, id) + if err != nil { + findings = append(findings, fmt.Errorf("%s prompt %q: %w", scope, id, err)) + continue + } + if profile != "" { + continue + } + if prompt.DefaultProfileID == nil || *prompt.DefaultProfileID == "" { + // A module may select a profile dynamically when constructing its + // actual request. No declared selection is not a missing resource. + continue + } + if _, err := engine.InspectProfile(ctx, *prompt.DefaultProfileID); err != nil { + findings = append(findings, fmt.Errorf("%s prompt %q default profile %q: %w", scope, id, *prompt.DefaultProfileID, err)) + } + } + } + input, _ := catalog.Inputs.Spec(resolved.Input.Module) + check(pipeline.StageInput, "", "", resolved.Input.Module, "", resolved.InputExecutionClass, resolved.Input.LLMProfile, input.Prompts) + check(pipeline.StageChunk, "", "", resolved.Chunk.Module, "", resolved.ChunkExecutionClass, resolved.Chunk.LLMProfile, resolved.ChunkPrompts) + steps := make(map[string]string) + for _, step := range resolved.Steps { + for _, lane := range step.ArtifactLanes { + steps[lane.ID] = step.ID + check(pipeline.StageExtract, step.ID, lane.ID, lane.Extract.Module, "", lane.ExtractExecutionClass, lane.Extract.LLMProfile, lane.ExtractPrompts) + check(pipeline.StageMerge, step.ID, lane.ID, lane.Merge.Module, "", lane.MergeExecutionClass, lane.Merge.LLMProfile, lane.MergePrompts) + check(pipeline.StageNormalize, step.ID, lane.ID, lane.Normalize.Module, "", lane.NormalizeExecutionClass, lane.Normalize.LLMProfile, lane.NormalizePrompts) + } + } + for _, chain := range resolved.ValidatorChains { + for _, validator := range chain.Validators { + check(chain.Stage, steps[chain.LaneID], chain.LaneID, chain.ModuleKey, validator.Binding.Module, validator.ExecutionClass, validator.Binding.LLMProfile, validator.Prompts) + } + } + output, _ := catalog.Outputs.Spec(resolved.Output.Module) + check(pipeline.StageOutput, "", "", resolved.Output.Module, "", resolved.OutputExecutionClass, resolved.Output.LLMProfile, output.Prompts) + if err := ctx.Err(); err != nil { + findings = append(findings, err) + } + return errors.Join(findings...) +} diff --git a/internal/preflight/preflight_test.go b/internal/preflight/preflight_test.go new file mode 100644 index 00000000..5903dcc9 --- /dev/null +++ b/internal/preflight/preflight_test.go @@ -0,0 +1,163 @@ +package preflight + +import ( + "context" + "errors" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" + "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" + "go.maximumdirect.net/configkit" + "go.maximumdirect.net/promptkit" +) + +type noGeneration struct{ t *testing.T } + +func (client noGeneration) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + client.t.Fatal("offline preflight invoked a provider") + return nil, nil +} + +func inspectionEngine(t *testing.T) *promptkit.Engine { + t.Helper() + engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: []configkit.Input{promptkit.SettingsInput(configkit.Bytes("offline resources", []byte(`profiles: + - id: partial + model: fixture + api_key_env: NOTARIUS_PREFLIGHT_MUST_NOT_REQUIRE_THIS_CREDENTIAL + api_key_required: true +prompts: + - id: initial + default_profile: partial + inputs: [{name: source, required: true}] + messages: [{role: user, content: '{{input "source"}}'}] + output: {format: text, validation_mode: none} + - id: correction + default_profile: partial + messages: [{role: user, content: 'Correct the previous response.'}] + output: {format: text, validation_mode: none} + - id: validator + default_profile: partial + messages: [{role: user, content: 'Assess.'}] + output: {format: text, validation_mode: none} + - id: unknown-default + default_profile: missing-profile + messages: [{role: user, content: 'Select a profile.'}] + output: {format: text, validation_mode: none} + - id: no-default + messages: [{role: user, content: 'Select a profile.'}] + output: {format: text, validation_mode: none} +`)))}}}}}, promptkit.WithBundledOpenRouter(false), promptkit.WithLLMClient(noGeneration{t})) + if err != nil { + t.Fatal(err) + } + return engine +} + +func resolvedFixture() pipeline.ResolvedPipeline { + return pipeline.ResolvedPipeline{ID: "campaign", Steps: []pipeline.ResolvedPipelineStep{{ID: "identify", ArtifactLanes: []pipeline.ResolvedArtifactLane{{ID: "characters", Extract: pipeline.Binding("character-producer"), ExtractExecutionClass: contracts.ExecutionClassLLMBacked, ExtractPrompts: []string{"initial", "correction"}}}}}, + ValidatorChains: []pipeline.ResolvedValidatorChain{{Stage: pipeline.StageExtract, LaneID: "characters", ModuleKey: "character-producer", Validators: []pipeline.ResolvedValidator{{Binding: pipeline.Binding("domain-validator"), ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{"validator"}}}}}, + } +} + +func TestOfflinePreflightAcceptsPartialDefaultsWithoutExecutionInputs(t *testing.T) { + engine := inspectionEngine(t) + resolved := resolvedFixture() + resolved.Steps[0].ArtifactLanes[0].ExtractReferences.Bindings = []pipeline.ReferenceBinding{{Source: "/not-present/reference.json"}} + if !RequiresEngine(resolved) { + t.Fatal("selected LLM roles were missed") + } + if err := Validate(context.Background(), resolved, pipeline.ModuleCatalog{}, engine); err != nil { + t.Fatalf("partial profile or absent execution inputs rejected: %v", err) + } +} + +func TestOfflinePreflightReportsEveryKnownResourceWithScopeAndCause(t *testing.T) { + resolved := resolvedFixture() + lane := &resolved.Steps[0].ArtifactLanes[0] + lane.Extract.LLMProfile = "missing-explicit" + lane.ExtractPrompts = []string{"missing-initial", "missing-correction"} + resolved.ValidatorChains[0].Validators[0].Prompts = []string{"missing-validator"} + err := Validate(context.Background(), resolved, pipeline.ModuleCatalog{}, inspectionEngine(t)) + if !errors.Is(err, promptkit.ErrPromptNotFound) || !errors.Is(err, promptkit.ErrProfileNotFound) { + t.Fatalf("native causes lost: %v", err) + } + for _, value := range []string{"campaign", "identify", "characters", "extract", "character-producer", "domain-validator", "missing-explicit", "missing-initial", "missing-correction", "missing-validator"} { + if !strings.Contains(err.Error(), value) { + t.Fatalf("missing preflight context %q: %v", value, err) + } + } +} + +func TestOfflinePreflightChecksApplicableProfilesButNotExecutionReadiness(t *testing.T) { + engine := inspectionEngine(t) + for _, test := range []struct { + prompt, profile string + want error + }{ + {"unknown-default", "", promptkit.ErrProfileNotFound}, + {"no-default", "", nil}, + {"unknown-default", "partial", nil}, + {"no-default", "partial", nil}, + } { + resolved := resolvedFixture() + lane := &resolved.Steps[0].ArtifactLanes[0] + lane.ExtractPrompts, lane.Extract.LLMProfile = []string{test.prompt}, test.profile + err := Validate(context.Background(), resolved, pipeline.ModuleCatalog{}, engine) + if test.want == nil && err != nil || test.want != nil && !errors.Is(err, test.want) { + t.Fatalf("prompt=%q profile=%q: %v", test.prompt, test.profile, err) + } + } +} + +func TestOfflinePreflightLeavesDynamicSelectionsForRuntime(t *testing.T) { + resolved := resolvedFixture() + resolved.Steps[0].ArtifactLanes[0].ExtractPrompts = nil + resolved.ValidatorChains = nil + if err := Validate(context.Background(), resolved, pipeline.ModuleCatalog{}, inspectionEngine(t)); err != nil { + t.Fatal(err) + } +} + +func TestOfflinePreflightIncludesInputOutputAndNestedValidatorsWithoutConstruction(t *testing.T) { + catalog := pipeline.ModuleCatalog{Inputs: pipeline.NewInputAdapterRegistry(), Outputs: pipeline.NewOutputEncoderRegistry()} + for _, stage := range []pipeline.ModuleStage{pipeline.StageInput, pipeline.StageOutput} { + spec := pipeline.ModuleSpec{Key: string(stage), Stage: stage, ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{"missing-" + string(stage)}} + var err error + if stage == pipeline.StageInput { + err = catalog.Inputs.RegisterWithSpec(spec, func() (contracts.InputAdapter, error) { t.Fatal("preflight constructed input"); return nil, nil }) + } else { + err = catalog.Outputs.RegisterWithSpec(spec, func() (contracts.OutputEncoder, error) { t.Fatal("preflight constructed output"); return nil, nil }) + } + if err != nil { + t.Fatal(err) + } + } + resolved := pipeline.ResolvedPipeline{ID: "io", Input: pipeline.Binding("input"), InputExecutionClass: contracts.ExecutionClassLLMBacked, Output: pipeline.Binding("output"), OutputExecutionClass: contracts.ExecutionClassLLMBacked} + err := Validate(context.Background(), resolved, catalog, inspectionEngine(t)) + if !errors.Is(err, promptkit.ErrPromptNotFound) || !strings.Contains(err.Error(), "missing-input") || !strings.Contains(err.Error(), "missing-output") { + t.Fatalf("input/output declarations were missed: %v", err) + } + resolved = resolvedFixture() + resolved.Steps[0].ArtifactLanes[0].ExtractExecutionClass = contracts.ExecutionClassDeterministic + if !RequiresEngine(resolved) { + t.Fatal("nested validator omitted from engine selection") + } +} + +func TestDeterministicPreflightNeedsNoEngineAndCancellationStopsInspection(t *testing.T) { + if RequiresEngine(pipeline.ResolvedPipeline{}) { + t.Fatal("deterministic composition requires engine") + } + if err := Validate(context.Background(), pipeline.ResolvedPipeline{}, pipeline.ModuleCatalog{}, nil); err != nil { + t.Fatal(err) + } + if err := Validate(context.Background(), resolvedFixture(), pipeline.ModuleCatalog{}, nil); !errors.Is(err, promptkit.ErrInvalidConfig) { + t.Fatalf("missing engine: %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := Validate(ctx, resolvedFixture(), pipeline.ModuleCatalog{}, nil); !errors.Is(err, context.Canceled) { + t.Fatalf("cancellation lost: %v", err) + } +}