package cli import ( "bytes" "context" "encoding/json" "errors" "os" "path/filepath" "reflect" "strings" "testing" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "go.maximumdirect.net/commandkit" "go.maximumdirect.net/promptkit" ) const inspectionConfiguration = `version: 6 output: {directory: output-must-not-exist} cache_directory: cache-must-not-exist debug: {directory: debug-must-not-exist} pipelines: demo: input: {module: fixture/input} chunk: {module: fixture/chunk} output: {module: fixture/output} steps: - id: build artifacts: notes: extract: module: fixture/extract references: {context: {path: reference-must-not-exist.txt}} validators: [{module: fixture/validator}] merge: {module: fixture/merge} normalize: {module: fixture/normalize} promptkit: profiles: - id: partial model: fixture-model api_key_env: NOTARIUS_INSPECTION_NO_CREDENTIAL_REQUIRED api_key_required: true prompts: - id: inspect-initial default_profile: partial inputs: [{name: source, required: true}] messages: [{role: user, content: '{{input "source"}}'}] output: {format: text, validation_mode: none} - id: inspect-correction default_profile: partial messages: [{role: user, content: Correct.}] output: {format: text, validation_mode: none} - id: inspect-validator default_profile: partial messages: [{role: user, content: Assess.}] output: {format: text, validation_mode: none} ` type inspectionArtifact struct { Value string `json:"value"` } type inspectionCodec struct{} func (inspectionCodec) Kind() contracts.ArtifactKind { return "test/inspection" } func (inspectionCodec) MediaType() string { return "application/json" } func (inspectionCodec) Schema() contracts.ArtifactSchema { return contracts.ArtifactSchema{ID: "test.inspection", Name: "inspection", Version: "v1", JSONSchema: []byte(`{"type":"object","properties":{"value":{"type":"string"}},"required":["value"]}`)} } func (inspectionCodec) Encode(value inspectionArtifact) ([]byte, error) { return json.Marshal(value) } func (codec inspectionCodec) EncodeCandidate(value inspectionArtifact) ([]byte, error) { return codec.Encode(value) } func (codec inspectionCodec) DecodeCandidate(content []byte) (inspectionArtifact, error) { return codec.Decode(content) } func (inspectionCodec) Decode(content []byte) (inspectionArtifact, error) { var value inspectionArtifact err := json.Unmarshal(content, &value) return value, err } func inspectionCatalog(t *testing.T, class contracts.ExecutionClass) pipeline.ModuleCatalog { t.Helper() catalog := pipeline.ModuleCatalog{Inputs: pipeline.NewInputAdapterRegistry(), Chunkers: pipeline.NewChunkerRegistry(), ArtifactCodecs: pipeline.NewArtifactCodecRegistry(), Extractors: pipeline.NewExtractorRegistry(), Mergers: pipeline.NewMergerRegistry(), Normalizers: pipeline.NewNormalizerRegistry(), Validators: pipeline.NewValidatorRegistry(), ValidatorChains: pipeline.NewValidatorChainRegistry(), Outputs: pipeline.NewOutputEncoderRegistry()} check := func(err error) { t.Helper() if err != nil { t.Fatal(err) } } spec := func(stage pipeline.ModuleStage, key string) pipeline.ModuleSpec { return pipeline.ModuleSpec{Key: key, Stage: stage, ExecutionClass: contracts.ExecutionClassDeterministic} } check(catalog.Inputs.RegisterWithSpec(spec(pipeline.StageInput, "fixture/input"), func() (contracts.InputAdapter, error) { t.Fatal("inspection constructed input"); return nil, nil })) check(catalog.Chunkers.RegisterWithSpec(spec(pipeline.StageChunk, "fixture/chunk"), func() (contracts.Chunker, error) { t.Fatal("inspection constructed chunker"); return nil, nil })) check(catalog.Outputs.RegisterWithSpec(spec(pipeline.StageOutput, "fixture/output"), func() (contracts.OutputEncoder, error) { t.Fatal("inspection constructed output"); return nil, nil })) check(pipeline.RegisterArtifactCodec(catalog.ArtifactCodecs, inspectionCodec{})) extract := spec(pipeline.StageExtract, "fixture/extract") extract.ArtifactKind, extract.ExecutionClass, extract.Prompts = "test/inspection", class, []string{"inspect-initial", "inspect-correction"} extract.ReferenceSlots = []contracts.ReferenceSlot{{Name: "context", Required: true, AcceptedMediaTypes: []string{"text/plain"}}} check(pipeline.RegisterExtractor(catalog.Extractors, extract, func() (contracts.Extractor[inspectionArtifact], error) { t.Fatal("inspection constructed extractor") return nil, nil })) merge := spec(pipeline.StageMerge, "fixture/merge") merge.ArtifactKind = "test/inspection" check(pipeline.RegisterMerger(catalog.Mergers, merge, func() (contracts.Merger[inspectionArtifact], error) { t.Fatal("inspection constructed merger") return nil, nil })) normalize := spec(pipeline.StageNormalize, "fixture/normalize") normalize.ArtifactKind = "test/inspection" check(pipeline.RegisterNormalizer(catalog.Normalizers, normalize, func() (contracts.Normalizer[inspectionArtifact], error) { t.Fatal("inspection constructed normalizer") return nil, nil })) check(pipeline.RegisterTypedValidator(catalog.Validators, "test/inspection", pipeline.ValidatorSpec{Key: "fixture/validator", ExecutionClass: class, Prompts: []string{"inspect-validator"}}, func() (contracts.TypedValidator[inspectionArtifact], error) { t.Fatal("inspection constructed validator") return nil, nil })) return catalog } type inspectionProvider struct{ t *testing.T } func (provider inspectionProvider) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { provider.t.Fatal("inspection called provider") return nil, nil } func writeInspectionConfiguration(t *testing.T, content string) string { t.Helper() file := filepath.Join(t.TempDir(), "config.yml") if err := os.WriteFile(file, []byte(content), 0600); err != nil { t.Fatal(err) } return file } func newInspectionApplication(t *testing.T, catalog func() (pipeline.ModuleCatalog, error)) (*commandkit.App, **nativeLoaders) { t.Helper() var last *nativeLoaders app, err := commandkit.New(commandkit.Identity{Name: "notarius"}, pipelineCommands(func() *nativeLoaders { last = &nativeLoaders{engineOptions: []promptkit.Option{promptkit.WithLLMClient(inspectionProvider{t})}} return last }, catalog)) if err != nil { t.Fatal(err) } return app, &last } func TestPipelineListOnlyEnumeratesCapturedConfiguration(t *testing.T) { content := strings.Replace(inspectionConfiguration, " demo:", " zeta:", 1) content = strings.Replace(content, "promptkit:", " alpha:\n input: {module: unavailable}\n artifacts:\n value:\n extract: {module: unavailable}\npromptkit:", 1) file := writeInspectionConfiguration(t, content) app, state := newInspectionApplication(t, func() (pipeline.ModuleCatalog, error) { t.Fatal("list constructed a catalog") return pipeline.ModuleCatalog{}, nil }) var out, diagnostics bytes.Buffer outcome := app.Execute(context.Background(), []string{"pipeline", "list", "--config", file, "--format", "json"}, commandkit.Streams{Out: &out, Err: &diagnostics}) var ids []string if outcome.Status != 0 || json.Unmarshal(out.Bytes(), &ids) != nil || !reflect.DeepEqual(ids, []string{"alpha", "zeta"}) { t.Fatalf("list=%#v %s %s", outcome, out.String(), diagnostics.String()) } if len((*state).selections) != 1 || (*state).selections[0].engine != nil { t.Fatal("list expanded native resources") } } func TestPipelineShowResolvesCompositionWithoutEngineOrExecutionInputs(t *testing.T) { file := writeInspectionConfiguration(t, inspectionConfiguration) catalog := inspectionCatalog(t, contracts.ExecutionClassLLMBacked) app, state := newInspectionApplication(t, func() (pipeline.ModuleCatalog, error) { return catalog, nil }) var out, diagnostics bytes.Buffer outcome := app.Execute(context.Background(), []string{"pipeline", "show", "demo", "--config", file, "--format", "json"}, commandkit.Streams{Out: &out, Err: &diagnostics}) var report pipelineDescription if err := json.Unmarshal(out.Bytes(), &report); err != nil || outcome.Status != 0 { t.Fatalf("show=%#v %s %s %v", outcome, out.String(), diagnostics.String(), err) } if report.Pipeline != "demo" || len(report.Steps) != 1 || report.Steps[0].ID != "build" || len(report.Steps[0].Lanes) != 1 || report.Steps[0].Lanes[0].Extract.Module != "fixture/extract" || len(report.Steps[0].Lanes[0].Extract.References) != 1 || len(report.Validators) == 0 { t.Fatalf("missing resolved composition: %#v", report) } if (*state).selections[0].engine != nil { t.Fatal("show expanded profiles by constructing engine") } assertInspectionNoRunStorage(t, file) } func TestPipelineValidateChecksNativeDeclarationsOfflineAndUsesOneLoad(t *testing.T) { for _, class := range []contracts.ExecutionClass{contracts.ExecutionClassDeterministic, contracts.ExecutionClassLLMBacked} { file := writeInspectionConfiguration(t, inspectionConfiguration) catalog := inspectionCatalog(t, class) app, state := newInspectionApplication(t, func() (pipeline.ModuleCatalog, error) { return catalog, nil }) var out, diagnostics bytes.Buffer outcome := app.Execute(context.Background(), []string{"pipeline", "validate", "demo", "--config", file}, commandkit.Streams{Out: &out, Err: &diagnostics}) if outcome.Status != 0 || !strings.Contains(out.String(), "demo") { t.Fatalf("validate=%#v %s %s", outcome, out.String(), diagnostics.String()) } if len((*state).selections) != 1 || ((*state).selections[0].engine != nil) != (class == contracts.ExecutionClassLLMBacked) { t.Fatal("validation loaded twice or built an unnecessary engine") } assertInspectionNoRunStorage(t, file) } } func TestPipelineValidateRejectsKnownMissingResourcesWithScope(t *testing.T) { for _, id := range []string{"inspect-initial", "inspect-correction", "inspect-validator"} { file := writeInspectionConfiguration(t, strings.Replace(inspectionConfiguration, "id: "+id, "id: unused-"+id, 1)) catalog := inspectionCatalog(t, contracts.ExecutionClassLLMBacked) app, _ := newInspectionApplication(t, func() (pipeline.ModuleCatalog, error) { return catalog, nil }) var out, diagnostics bytes.Buffer outcome := app.Execute(context.Background(), []string{"pipeline", "validate", "demo", "--config", file}, commandkit.Streams{Out: &out, Err: &diagnostics}) if outcome.Status != 2 || !errors.Is(outcome.Cause, promptkit.ErrPromptNotFound) || !strings.Contains(diagnostics.String(), id) || !strings.Contains(diagnostics.String(), "build") || !strings.Contains(diagnostics.String(), "notes") { t.Fatalf("known dependency=%#v %s", outcome, diagnostics.String()) } } } func TestPipelineInspectionUsesNativeArgumentsSelectionsAndDelivery(t *testing.T) { file := writeInspectionConfiguration(t, inspectionConfiguration) catalog := inspectionCatalog(t, contracts.ExecutionClassLLMBacked) app, _ := newInspectionApplication(t, func() (pipeline.ModuleCatalog, error) { return catalog, nil }) for _, test := range []struct { args []string status int }{ {[]string{"pipeline", "show"}, 2}, {[]string{"pipeline", "show", "demo", "--config", "", "--config", file}, 2}, {[]string{"pipeline", "show", "unknown", "--config", file}, 2}, {[]string{"pipeline", "show", "demo", "--config", file, "--set", `/pipelines/demo/steps/0/artifacts/notes/extract/options={"unsupported":true}`}, 2}, {[]string{"pipeline", "validate", "demo", "--config", file, "--set", `/pipelines/demo/llm_profile="missing-profile"`}, 2}, {[]string{"pipeline", "show", "demo", "--config", file, "--set", `/pipelines/demo/steps/0/artifacts/notes/extract/references=null`}, 2}, {[]string{"pipeline", "list", "--config", file, "--set", `/pipelines/demo/input/module="another-input"`}, 0}, {[]string{"pipeline", "validate", "demo", "--config", file, "--quiet"}, 0}, {[]string{"pipeline", "--help"}, 0}, } { var out, diagnostics bytes.Buffer outcome := app.Execute(context.Background(), test.args, commandkit.Streams{Out: &out, Err: &diagnostics}) if outcome.Status != test.status { t.Fatalf("%v: %#v %s", test.args, outcome, diagnostics.String()) } if test.args[len(test.args)-1] == "--quiet" && out.Len() != 0 { t.Fatal("quiet pipeline inspection wrote stdout") } } } type inspectionFailWriter struct{ err error } func (writer inspectionFailWriter) Write([]byte) (int, error) { return 0, writer.err } func TestPipelineInspectionReportFailureRemainsOperational(t *testing.T) { file := writeInspectionConfiguration(t, inspectionConfiguration) app, _ := newInspectionApplication(t, func() (pipeline.ModuleCatalog, error) { t.Fatal("list loaded catalog") return pipeline.ModuleCatalog{}, nil }) primary := errors.New("inspection stdout delivery failed") var diagnostics bytes.Buffer outcome := app.Execute(context.Background(), []string{"pipeline", "list", "--config", file}, commandkit.Streams{Out: inspectionFailWriter{primary}, Err: &diagnostics}) if outcome.Status != 1 || outcome.Cause != nil || !errors.Is(outcome.Presentation.Output.WriteError, primary) { t.Fatalf("delivery failure classified as invalid configuration: %#v %s", outcome, diagnostics.String()) } } func assertInspectionNoRunStorage(t *testing.T, file string) { t.Helper() entries, err := os.ReadDir(filepath.Dir(file)) if err != nil || len(entries) != 1 || entries[0].Name() != "config.yml" { t.Fatalf("inspection read/materialized inputs or created storage: %v %v", entries, err) } }