package cli import ( "bytes" "context" "encoding/json" "fmt" "net/url" "os" "path/filepath" "strings" "sync" "testing" "time" "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" units "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/chunk/units" jsonoutput "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/output/json" "go.maximumdirect.net/commandkit" "go.maximumdirect.net/promptkit" ) type runProbe struct { mu sync.Mutex requests []promptkit.GenerateRequest prepared []promptkit.PreparedRun references []contracts.ReferenceItem engines []*promptkit.Engine parsed int produced int correctFirst bool } func (probe *runProbe) Generate(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { probe.mu.Lock() defer probe.mu.Unlock() probe.requests = append(probe.requests, request) value := "accepted" if request.Prompt.Messages[len(request.Prompt.Messages)-1].Content != "Assess." { probe.produced++ if probe.correctFirst && probe.produced == 1 { value = "rejected" } } return &promptkit.GenerateResponse{Content: fmt.Sprintf(`{"value":%q}`, value)}, nil } func (probe *runProbe) onPrepared(_ context.Context, details promptkit.PreparedRun) { probe.mu.Lock() defer probe.mu.Unlock() probe.prepared = append(probe.prepared, details) } type runInputAdapter struct{ probe *runProbe } func (*runInputAdapter) Key() string { return "fixture/input" } func (adapter *runInputAdapter) Parse(_ context.Context, request contracts.ParseRequest) (*source.SourceDocument, error) { adapter.probe.mu.Lock() adapter.probe.parsed++ adapter.probe.mu.Unlock() document := &source.SourceDocument{ID: "source", Kind: "fixture", Format: "text", Units: []source.SourceUnit{{ID: 1, Kind: "text", Text: string(request.Raw), Ref: source.SourceRef{SourceID: "source", StartUnitID: 1, EndUnitID: 1}}}} var err error document.Digest, err = source.DigestDocument(document) return document, err } type runProducer struct { key string engine *promptkit.Engine probe *runProbe } func (producer *runProducer) Key() string { return producer.key } func (*runProducer) ReferenceSlots() []contracts.ReferenceSlot { return runReferenceSlots() } func runReferenceSlots() []contracts.ReferenceSlot { return []contracts.ReferenceSlot{ {Name: "context", Required: true, AcceptedMediaTypes: []string{"text/plain", "application/json"}, AcceptedArtifactKinds: []contracts.ArtifactKind{"test/inspection"}}, {Name: "secondary", AcceptedMediaTypes: []string{"text/plain"}}, } } func (producer *runProducer) Extract(ctx context.Context, request contracts.TypedExtractionRequest[inspectionArtifact]) (contracts.TypedExtractionResult[inspectionArtifact], error) { content := string(request.SourceInput.Content) for _, slot := range request.References.Slots { for _, item := range slot.Items { content += "\n" + string(item.Content) producer.probe.mu.Lock() producer.probe.references = append(producer.probe.references, item) producer.probe.mu.Unlock() } } initial := promptkit.RunRequest{PromptID: "inspect-initial", ProfileID: request.LLMProfile, SessionID: request.SessionID, Inputs: map[string]promptkit.ArtifactRef{"source": promptkit.Inline(content)}, OnPrepared: producer.probe.onPrepared} value, outcome, err := pipeline.RunProducer(ctx, producer.engine, request.Operation, initial, func(_ context.Context, result *promptkit.RunResult) (*inspectionArtifact, []string, error) { var value inspectionArtifact err := json.Unmarshal([]byte(result.RawOutput), &value) return &value, nil, err }, func(context.Context, []string) (promptkit.RunRequest, error) { return promptkit.RunRequest{PromptID: "inspect-correction", ProfileID: request.LLMProfile, SessionID: request.SessionID, OnPrepared: producer.probe.onPrepared}, nil }, nil) return contracts.TypedExtractionResult[inspectionArtifact]{Value: value, Outcome: outcome}, err } func (producer *runProducer) Merge(_ context.Context, request contracts.TypedMergeRequest[inspectionArtifact]) (contracts.TypedMergeResult[inspectionArtifact], error) { return contracts.TypedMergeResult[inspectionArtifact]{Value: request.ExtractOutputs[0].Value}, nil } func (producer *runProducer) Normalize(_ context.Context, request contracts.TypedNormalizeRequest[inspectionArtifact]) (contracts.TypedNormalizeResult[inspectionArtifact], error) { return contracts.TypedNormalizeResult[inspectionArtifact]{Value: request.MergeOutput.Value}, nil } type runValidator struct { engine *promptkit.Engine probe *runProbe } func (*runValidator) Name() string { return "fixture/validator" } func (*runValidator) ExecutionClass() contracts.ExecutionClass { return contracts.ExecutionClassLLMBacked } func (validator *runValidator) Validate(ctx context.Context, request contracts.TypedValidationRequest[inspectionArtifact]) (contracts.ValidationResult, error) { result, err := validator.engine.Run(ctx, promptkit.RunRequest{PromptID: "inspect-validator", ProfileID: request.LLMProfile, SessionID: request.SessionID, Execution: request.Execution.Execution, TransientRetries: request.Execution.TransientRetries, Observer: request.Execution.Observer, OnPrepared: validator.probe.onPrepared}) if err != nil { return contracts.ValidationResult{}, err } if !result.Validation.IsValid { return contracts.ValidationResult{}, fmt.Errorf("native validator contract failed") } if request.Value.Value != "accepted" { return contracts.ValidationResult{ReasonCode: "rejected", CorrectionGuidance: "Return accepted."}, nil } return contracts.ValidationResult{Approved: true}, nil } func runRegistries(t *testing.T, probe *runProbe) pipeline.Registries { t.Helper() registries := pipeline.Registries{Inputs: pipeline.NewInputAdapterRegistry(), Chunkers: pipeline.NewChunkerRegistry(), ArtifactCodecs: pipeline.NewArtifactCodecRegistry(), Extractors: pipeline.NewExtractorRegistry(), Mergers: pipeline.NewMergerRegistry(), Normalizers: pipeline.NewNormalizerRegistry(), Validators: pipeline.NewValidatorRegistry(), ValidatorChains: pipeline.NewValidatorChainRegistry(), Outputs: pipeline.NewOutputEncoderRegistry()} check := func(err error) { t.Helper() if err != nil { t.Fatal(err) } } check(registries.Inputs.RegisterWithSpec(pipeline.ModuleSpec{Key: "fixture/input", Stage: pipeline.StageInput, ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.InputAdapter, error) { return &runInputAdapter{probe}, nil })) check(units.Register(registries.Chunkers)) check(jsonoutput.Register(registries.Outputs)) check(pipeline.RegisterArtifactCodec(registries.ArtifactCodecs, inspectionCodec{})) slots := runReferenceSlots() check(pipeline.RegisterExtractorBuilder(registries.Extractors, pipeline.ModuleSpec{Key: "fixture/extract", Stage: pipeline.StageExtract, ArtifactKind: "test/inspection", ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{"inspect-initial", "inspect-correction"}, ReferenceSlots: slots}, func(options map[string]any) error { return pipeline.RejectUnknownOptions(options) }, func(request pipeline.BuildRequest) (contracts.Extractor[inspectionArtifact], error) { probe.engines = append(probe.engines, request.Dependencies.Engine) return &runProducer{key: "fixture/extract", engine: request.Dependencies.Engine, probe: probe}, nil })) check(pipeline.RegisterMerger(registries.Mergers, pipeline.ModuleSpec{Key: "fixture/merge", Stage: pipeline.StageMerge, ArtifactKind: "test/inspection", ExecutionClass: contracts.ExecutionClassDeterministic}, func() (contracts.Merger[inspectionArtifact], error) { return &runProducer{key: "fixture/merge", probe: probe}, nil })) check(pipeline.RegisterNormalizer(registries.Normalizers, pipeline.ModuleSpec{Key: "fixture/normalize", Stage: pipeline.StageNormalize, ArtifactKind: "test/inspection", ExecutionClass: contracts.ExecutionClassDeterministic, Provides: []string{"normalized"}}, func() (contracts.Normalizer[inspectionArtifact], error) { return &runProducer{key: "fixture/normalize", probe: probe}, nil })) check(pipeline.RegisterTypedValidatorBuilder(registries.Validators, "test/inspection", pipeline.ValidatorSpec{Key: "fixture/validator", ExecutionClass: contracts.ExecutionClassLLMBacked, Prompts: []string{"inspect-validator"}}, func(options map[string]any) error { return pipeline.RejectUnknownOptions(options) }, func(request pipeline.BuildRequest) (contracts.TypedValidator[inspectionArtifact], error) { probe.engines = append(probe.engines, request.Dependencies.Engine) return &runValidator{request.Dependencies.Engine, probe}, nil })) return registries } func runFixture(t *testing.T, probe *runProbe, configure ...func(string) string) (*commandkit.App, string, string, **nativeLoaders) { t.Helper() root := t.TempDir() configuration := strings.ReplaceAll(inspectionConfiguration, "fixture/chunk", "generic") configuration = strings.ReplaceAll(configuration, "fixture/output", "json") configuration = strings.Replace(configuration, " api_key_env: NOTARIUS_INSPECTION_NO_CREDENTIAL_REQUIRED\n api_key_required: true", " endpoint: https://example.invalid/v1\n api_key_required: false", 1) configuration = strings.Replace(configuration, " prompts:", ` - id: chosen endpoint: https://example.invalid/v1 model: chosen-model reasoning_effort: high temperature: 0.8 api_key_required: false prompts:`, 1) for _, update := range configure { configuration = update(configuration) } file := filepath.Join(root, "config.yml") input := filepath.Join(root, "input.txt") for path, content := range map[string]string{file: configuration, input: "source transcript", filepath.Join(root, "reference-must-not-exist.txt"): "configured reference"} { if err := os.WriteFile(path, []byte(content), 0600); err != nil { t.Fatal(err) } } registries := runRegistries(t, probe) options := Options{Now: func() time.Time { return time.Unix(42, 0) }} runs := 0 options.RunIDGenerator = func(time.Time) (string, error) { runs++; return fmt.Sprintf("run-%d", runs), nil } var state *nativeLoaders newState := func() *nativeLoaders { state = &nativeLoaders{engineOptions: []promptkit.Option{promptkit.WithLLMClient(probe)}} return state } app, err := commandkit.New(commandkit.Identity{Name: "notarius"}, pipelineCommands(newState, func() (pipeline.ModuleCatalog, error) { return catalogFromRegistries(registries), nil }, pipelineRunCommand("development", newState, func() (pipeline.Registries, error) { return registries, nil }, options)), commandkit.Alias("run", "pipeline run")) if err != nil { t.Fatal(err) } return app, file, input, &state } func executeRunFixture(t *testing.T, app *commandkit.App, args ...string) commandkit.Outcome { t.Helper() var out, diagnostics bytes.Buffer outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &out, Err: &diagnostics}) if outcome.Status != 0 { writeConfigLoadError(&diagnostics, outcome.Cause) t.Fatalf("%v: %#v stderr=%s", args, outcome, diagnostics.String()) } return outcome } func TestNativeRunRoutesShareOneHandlerAndApplyOverridesToEveryRole(t *testing.T) { probe := &runProbe{correctFirst: true} app, file, input, state := runFixture(t, probe) executeRunFixture(t, app, "run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--llm-profile", "chosen", "--llm-model", "operator-model", "--llm-session-id", "operator-session", "--llm-reasoning-effort=", "--llm-temperature=0", "--llm-transient-retries=0") if probe.produced != 2 || len(probe.requests) != 4 || probe.parsed != 1 { t.Fatalf("native operation/correction/validation = %d requests=%d parse=%d", probe.produced, len(probe.requests), probe.parsed) } for _, request := range probe.requests { if request.Target.Model != "operator-model" || request.Prompt.SessionID != "operator-session" || request.Target.ReasoningEffort != "" || request.Target.Temperature != 0 || !request.TargetPresence.Temperature { t.Fatalf("override missing on a native role: %#v", request) } } for _, prepared := range probe.prepared { if prepared.SelectedProfileID != "chosen" { t.Fatalf("profile override missing: %s", prepared.SelectedProfileID) } } if len((*state).selections) != 1 || len(probe.engines) != 2 || probe.engines[0] == nil || probe.engines[0] != probe.engines[1] || probe.engines[0] != (*state).selections[0].engine { t.Fatal("run used a second load or engine") } if (*state).selections[0].result.Config.Pipelines["demo"].LLMProfile != nil { t.Fatal("request override mutated captured configuration") } executeRunFixture(t, app, "pipeline", "run", "demo", "--config", file, "--input-file", input, "--cache", "bypass") for _, id := range []string{"run-1", "run-2"} { content, err := os.ReadFile(filepath.Join(filepath.Dir(file), "output-must-not-exist", id, "lanes", "notes.json")) if err != nil || !strings.Contains(string(content), "accepted") { t.Fatalf("published artifact %s: %s %v", id, content, err) } } } func TestNativeRunSessionIdentityIgnoresReferenceChangesAndHonorsEmptyOverride(t *testing.T) { probe := &runProbe{} app, file, input, _ := runFixture(t, probe) first := filepath.Join(t.TempDir(), "reference one.txt") second := filepath.Join(t.TempDir(), "reference two.txt") for path, text := range map[string]string{first: "first reference", second: "second reference"} { if err := os.WriteFile(path, []byte(text), 0600); err != nil { t.Fatal(err) } } executeRunFixture(t, app, "run", "demo", "--config", file, "--input-file", input, "--reference", "context="+first, "--cache", "bypass") session := probe.requests[0].Prompt.SessionID executeRunFixture(t, app, "run", "demo", "--config", file, "--input-file", input, "--reference", "context="+second, "--cache", "bypass") if session == "" || session != probe.requests[2].Prompt.SessionID || !strings.Contains(probe.requests[0].Prompt.Messages[0].Content, "first reference") || !strings.Contains(probe.requests[2].Prompt.Messages[0].Content, "second reference") { t.Fatal("reference selection changed generated session or failed to reach request") } executeRunFixture(t, app, "run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--set", `/pipelines/demo/llm_profile="chosen"`, "--llm-session-id=", "--llm-profile=") if probe.requests[4].Prompt.SessionID != "" || probe.prepared[4].SelectedProfileID != "partial" { t.Fatal("explicit empty native selections were replaced by application defaults") } } func TestNativeRunValidatesEverySuppliedValueBeforeWork(t *testing.T) { probe := &runProbe{} app, file, input, _ := runFixture(t, probe) for _, args := range [][]string{ {"run", "demo", "--config", file}, {"run", "--config", file, "--input-file", input}, {"run", "demo", "--config", file, "--input-file", "", "--input-file", input}, {"run", "demo", "--config", file, "--input-file", input, "--cache", "bad", "--cache", "auto"}, {"run", "demo", "--config", file, "--input-file", input, "--output-dir", "", "--output-dir", t.TempDir()}, {"run", "demo", "--config", file, "--input-file", input, "--reference", "invalid"}, {"run", "demo", "--config", file, "--input-file", input, "--without-reference", "context"}, {"run", "demo", "--config", file, "--input-file", input, "--llm-temperature=invalid", "--llm-temperature=0"}, {"run", "demo", "--config", file, "--input-file", input, "--only", "notes"}, {"run", "demo", "--config", file, "--input-file", input, "--json"}, {"run", "demo", "--config", file, "--input-file", input, "--resume"}, {"run", "demo", "--config", file, "--input-file", input, "--recompute-step", "build"}, {"run", "demo", "--config", file, "--input-file", input, "--chunk-cache", "auto"}, } { var out, diagnostics bytes.Buffer outcome := app.Execute(context.Background(), args, commandkit.Streams{Out: &out, Err: &diagnostics}) if outcome.Status != 2 { t.Fatalf("%v: %#v %s", args, outcome, diagnostics.String()) } } if len(probe.requests) != 0 || probe.parsed != 0 { t.Fatal("invalid invocation started source/model execution") } } func TestNativeRunDebugAndOutputOverridesRemainInvocationRelative(t *testing.T) { probe := &runProbe{} app, file, input, _ := runFixture(t, probe) output := t.TempDir() debug := t.TempDir() workingDirectory, err := os.Getwd() if err != nil { t.Fatal(err) } relativeOutput, err := filepath.Rel(workingDirectory, output) if err != nil { t.Fatal(err) } relativeDebug, err := filepath.Rel(workingDirectory, debug) if err != nil { t.Fatal(err) } executeRunFixture(t, app, "run", "demo", "--config", file, "--input-file", input, "--output-dir", relativeOutput, "--debug", "--debug-dir", relativeDebug, "--cache", "bypass") if _, err := os.Stat(filepath.Join(output, "run-1", "lanes", "notes.json")); err != nil { t.Fatal(err) } if _, err := os.Stat(filepath.Join(debug, "run-1", "outcome.json")); err != nil { t.Fatal(err) } executeRunFixture(t, app, "run", "demo", "--config", file, "--input-file", input, "--debug", "--debug-dir=", "--cache", "bypass") if _, err := os.Stat(filepath.Join(filepath.Dir(file), "debug-must-not-exist")); !os.IsNotExist(err) { t.Fatalf("explicit empty debug path allocated capture: %v", err) } executeRunFixture(t, app, "run", "demo", "--config", file, "--input-file", input, "--debug", "--debug=false", "--debug-dir", debug, "--cache", "bypass") if _, err := os.Stat(filepath.Join(debug, "run-3")); !os.IsNotExist(err) { t.Fatalf("explicit false debug selection allocated capture: %v", err) } } func TestNativeRunReferencesKeepSpecificityAndFileOrigins(t *testing.T) { probe := &runProbe{} app, file, input, _ := runFixture(t, probe, func(configuration string) string { return strings.Replace(configuration, "references: {context: {path: reference-must-not-exist.txt}}", "references: {context: {path: reference-must-not-exist.txt}, secondary: {path: never-read.txt}}", 1) }) workingDirectory, err := os.Getwd() if err != nil { t.Fatal(err) } paths := make(map[string]string) for _, name := range []string{"broad", "lane", "binding", "secondary"} { absolute := filepath.Join(t.TempDir(), name+" reference.txt") if err := os.WriteFile(absolute, []byte(name+" reference"), 0600); err != nil { t.Fatal(err) } paths[name], err = filepath.Rel(workingDirectory, absolute) if err != nil { t.Fatal(err) } } executeRunFixture(t, app, "run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--reference", "notes.extract.context="+paths["binding"], "--reference", "notes.context="+paths["lane"], "--reference", "context="+paths["broad"], "--reference", "secondary="+paths["secondary"]) if len(probe.references) != 2 { t.Fatalf("materialized references: %#v", probe.references) } for _, item := range probe.references { name := "binding" if string(item.Content) == "secondary reference" { name = "secondary" } absolute := filepath.Clean(filepath.Join(workingDirectory, paths[name])) if string(item.Content) != name+" reference" || item.BindingSource != contracts.ReferenceBindingSourceCLI || item.Origin.URI != (&url.URL{Scheme: "file", Path: absolute}).String() { t.Fatalf("CLI reference content/origin: %#v", item) } } probe.references = nil executeRunFixture(t, app, "run", "demo", "--config", file, "--input-file", input, "--cache", "bypass", "--without-reference", "secondary") configured := (&url.URL{Scheme: "file", Path: filepath.Join(filepath.Dir(file), "reference-must-not-exist.txt")}).String() if len(probe.references) != 1 || string(probe.references[0].Content) != "configured reference" || probe.references[0].BindingSource != contracts.ReferenceBindingSourceConfig || probe.references[0].Origin.URI != configured { t.Fatalf("config reference origin or optional unbind: %#v", probe.references) } } func TestNativeRunLiteralLanesAndFinalSingletonValues(t *testing.T) { probe := &runProbe{} lane := `{"extract":{"module":"fixture/extract","references":{"context":{"path":"reference-must-not-exist.txt"}},"validators":[{"module":"fixture/validator"}]},"merge":{"module":"fixture/merge"},"normalize":{"module":"fixture/normalize"}}` artifacts := `{"notes,extra":` + lane + `,"other":` + lane + `,"unselected":` + lane + `}` app, file, input, _ := runFixture(t, probe, func(configuration string) string { start := strings.Index(configuration, " steps:") end := strings.Index(configuration, "promptkit:") return configuration[:start] + " artifacts: " + artifacts + "\n" + configuration[end:] }) output := t.TempDir() executeRunFixture(t, app, "run", "--config", file, "--input-file", filepath.Join(t.TempDir(), "earlier-input.txt"), "--input-file", input, "--output-dir", filepath.Join(t.TempDir(), "earlier-output"), "--output-dir", output, "--llm-profile", "not-the-final-profile", "--llm-profile", "chosen", "--cache", "auto", "--cache", "bypass", "--lane", "notes,extra", "--lane", "other", "--", "demo") if probe.produced != 2 || probe.parsed != 1 { t.Fatalf("selected lanes or singleton input: produced=%d parsed=%d", probe.produced, probe.parsed) } files, err := os.ReadDir(filepath.Join(output, "run-1", "lanes")) if err != nil || len(files) != 2 { t.Fatalf("selected output files: %#v %v", files, err) } for _, prepared := range probe.prepared { if prepared.SelectedProfileID != "chosen" { t.Fatalf("final singleton profile not used: %q", prepared.SelectedProfileID) } } if _, err := os.Stat(filepath.Join(filepath.Dir(file), "cache-must-not-exist")); !os.IsNotExist(err) { t.Fatalf("final bypass value allocated cache: %v", err) } } func TestNativeRunProtectsGeneratedReferencesBeforeReadingFiles(t *testing.T) { probe := &runProbe{} app, file, input, _ := runFixture(t, probe) steps := `[{"id":"produce","artifacts":{"earlier":{"extract":{"module":"fixture/extract","references":{"context":{"path":"reference-must-not-exist.txt"}}},"merge":{"module":"fixture/merge"},"normalize":{"module":"fixture/normalize"}}}},{"id":"consume","artifacts":{"later":{"extract":{"module":"fixture/extract","references":{"context":{"artifact":{"step":"produce","lane":"earlier"}}}},"merge":{"module":"fixture/merge"},"normalize":{"module":"fixture/normalize"}}}}]` var out, diagnostics bytes.Buffer outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--set", "/pipelines/demo/steps=" + steps, "--reference", "context=never-read.txt"}, commandkit.Streams{Out: &out, Err: &diagnostics}) if outcome.Status != 2 || !strings.Contains(diagnostics.String(), "conflicting generated and external bindings") || len(probe.requests) != 0 || probe.parsed != 0 { t.Fatalf("generated/external conflict: %#v %s requests=%d parsed=%d", outcome, diagnostics.String(), len(probe.requests), probe.parsed) } } func TestNativeRunCapturesPreflightFailureBeforeSourceExecution(t *testing.T) { probe := &runProbe{} app, file, _, state := runFixture(t, probe) content, err := os.ReadFile(file) if err != nil { t.Fatal(err) } content = bytes.Replace(content, []byte("id: inspect-correction"), []byte("id: different-correction"), 1) if err := os.WriteFile(file, content, 0600); err != nil { t.Fatal(err) } var out, diagnostics bytes.Buffer outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", filepath.Join(t.TempDir(), "missing-input.txt"), "--debug"}, commandkit.Streams{Out: &out, Err: &diagnostics}) if outcome.Status != 2 || !strings.Contains(diagnostics.String(), "inspect-correction") || probe.parsed != 0 || len(probe.requests) != 0 || len(probe.engines) != 0 { t.Fatalf("preflight order: %#v %s", outcome, diagnostics.String()) } if (*state).selections[0].engine == nil { t.Fatal("preflight did not inspect the captured engine") } if _, err := os.Stat(filepath.Join(filepath.Dir(file), "debug-must-not-exist", "run-1", "outcome.json")); err != nil { t.Fatalf("failed preflight capture: %v", err) } } func TestNativeRunDebugAllocationFailurePreventsEngineConstruction(t *testing.T) { probe := &runProbe{} app, file, input, state := runFixture(t, probe) debug := filepath.Join(t.TempDir(), "not-a-directory") if err := os.WriteFile(debug, []byte("occupied"), 0600); err != nil { t.Fatal(err) } var out, diagnostics bytes.Buffer outcome := app.Execute(context.Background(), []string{"run", "demo", "--config", file, "--input-file", input, "--debug", "--debug-dir", debug}, commandkit.Streams{Out: &out, Err: &diagnostics}) if outcome.Status != 1 || probe.parsed != 0 || len(probe.requests) != 0 || len(probe.engines) != 0 || (*state).selections[0].engine != nil { t.Fatalf("capture allocation order: %#v %s", outcome, diagnostics.String()) } }