From f5573f7c8f2ddf0d69414a05c978c8d6cfe291e7 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Fri, 20 Nov 2026 05:17:05 +0000 Subject: [PATCH] Verify the complete native CLI cutover and record roadmap completion --- cmd/scriptorium/main_test.go | 88 +++++++++++++++++++ docs/roadmap/commandkit-adoption.md | 17 ++-- docs/roadmap/implementation.md | 38 ++++++++ docs/roadmap/narratio-feature-request.md | 5 +- internal/adapter/cli/delivery_test.go | 36 ++++++++ internal/adapter/cli/discovery_test.go | 105 ++++++++++++++++++++++- internal/adapter/cli/native_run_test.go | 3 + 7 files changed, 281 insertions(+), 11 deletions(-) diff --git a/cmd/scriptorium/main_test.go b/cmd/scriptorium/main_test.go index 41c7331..a9a6a9a 100644 --- a/cmd/scriptorium/main_test.go +++ b/cmd/scriptorium/main_test.go @@ -3,8 +3,20 @@ package main import ( "bytes" "context" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "runtime" + "slices" "strings" + "syscall" "testing" + "time" ) func TestProcessIdentityAndNativeStatus(t *testing.T) { @@ -23,3 +35,79 @@ func TestProcessIdentityAndNativeStatus(t *testing.T) { t.Fatalf("canceled status=%d diagnostics=%q", status, diagnostics.String()) } } + +func TestProcessSignalCancellation(t *testing.T) { + if os.Getenv("SCRIPTORIUM_SIGNAL_CHILD") == "1" { + separator := slices.Index(os.Args, "--") + if separator < 0 { + os.Exit(2) + } + os.Exit(runWithSignals(os.Args[separator+1:], os.Stdout, os.Stderr)) + } + if runtime.GOOS == "windows" { + t.Skip("process interrupt/SIGTERM integration is Unix-specific") + } + for _, signal := range []os.Signal{os.Interrupt, syscall.SIGTERM} { + t.Run(signal.String(), func(t *testing.T) { + started, stopped, release := make(chan struct{}), make(chan struct{}), make(chan struct{}) + provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + close(started) + select { + case <-r.Context().Done(): + close(stopped) + case <-release: + } + })) + defer provider.Close() + defer close(release) + root := t.TempDir() + framework := filepath.Join(root, "framework.yml") + content := fmt.Sprintf("profiles:\n - id: local\n endpoint: %q\n model: controlled\nprompts:\n - id: selected\n default_profile: local\n messages:\n - role: user\n content: Signal test\n output:\n format: text\n validation_mode: none\n", provider.URL+"/v1") + if err := os.WriteFile(framework, []byte(content), 0600); err != nil { + t.Fatal(err) + } + config := filepath.Join(root, "app.yml") + if err := os.WriteFile(config, []byte(fmt.Sprintf("promptkit:\n configuration: %q\ntransient_retries: 0\n", framework)), 0600); err != nil { + t.Fatal(err) + } + executable, err := os.Executable() + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + command := exec.CommandContext(ctx, executable, "-test.run=^TestProcessSignalCancellation$", "--", "run", "selected", "--config="+config, "--timeout-seconds=0") + command.Env = append(os.Environ(), "SCRIPTORIUM_SIGNAL_CHILD=1") + var output, diagnostics bytes.Buffer + command.Stdout, command.Stderr = &output, &diagnostics + if err := command.Start(); err != nil { + t.Fatal(err) + } + defer func() { + cancel() + if command.ProcessState == nil { + _ = command.Wait() + } + }() + select { + case <-started: + case <-ctx.Done(): + t.Fatal("child did not reach controlled provider") + } + if err := command.Process.Signal(signal); err != nil { + t.Fatal(err) + } + err = command.Wait() + var exit *exec.ExitError + if !errors.As(err, &exit) || exit.ExitCode() != 1 || output.Len() != 0 || diagnostics.Len() == 0 { + t.Fatalf("signal outcome: %v stdout=%q stderr=%q", err, output.String(), diagnostics.String()) + } + select { + case <-stopped: + case <-time.After(5 * time.Second): + t.Fatal("signal did not cancel provider request") + } + }) + } +} diff --git a/docs/roadmap/commandkit-adoption.md b/docs/roadmap/commandkit-adoption.md index 1364952..e76976d 100644 --- a/docs/roadmap/commandkit-adoption.md +++ b/docs/roadmap/commandkit-adoption.md @@ -2,8 +2,9 @@ ## Status And Intent -Initial roadmap, prepared 2026-11-19. The product direction is agreed; -implementation is underway. This document owns the target and product intent, +Implementation and assembled acceptance verification are complete; see the +[implementation plan](implementation.md) for evidence and selected releases. +Initial roadmap prepared 2026-11-19. This document owns the target and product intent, incorporating the decisions in the [migration notepad](commandkit-notepad.md). The notepad remains the discussion record. The [Narratio proposal](narratio-feature-request.md) supplies consumer @@ -26,13 +27,13 @@ report shapes, and exit conventions are not compatibility constraints. Release migration guidance replaces a compatibility layer. Implementation may proceed in reviewable increments; this does not require shipping both interfaces. -## Current Baseline And Upstream References +## Planning Baseline And Upstream References Reviewed Scriptorium baseline: `5f32a4f2adb2d382a6cfce3a787ada4576be7705`. -The module uses Go 1.27.0, PromptKit v0.23.3, and ConfigKit v1.2.1. CommandKit -is not yet a dependency. +At that baseline the module used Go 1.27.0, PromptKit v0.23.3, and ConfigKit +v1.2.1, without CommandKit. The following table records the starting point. -| Current implementation | Migration work | +| Baseline implementation | Migration work | | --- | --- | | `internal/adapter/cli` dispatches and parses commands, resolves configuration, builds engines and requests, writes diagnostics, and returns integer statuses. | Replace dispatch and shared CLI mechanics with CommandKit; retain application loading, request mapping, and effects through its extension points. | | `run` uses flag-based prompt selection and writes generated content; `--execution-report` writes an observer-derived Scriptorium report. | Adopt positional prompt selection, native request flags, and native result reporting. | @@ -44,8 +45,8 @@ is not yet a dependency. The [current CLI](../cli.md), [configuration contract](../config.md), [adapter internals](../internal/adapters.md), and -[subprocess contract](../integrations/subprocess.md) describe the starting -behavior, not a compatibility checklist for this migration. +[subprocess contract](../integrations/subprocess.md) now describe the implemented +native interface, not the historical starting point or a compatibility layer. The reviewed upstream combination is CommandKit v0.7.0, ckconfig v0.9.0, and ckprompt v0.10.0, documented at revision diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 631e9bb..b4f6952 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -723,6 +723,44 @@ current documentation links resolve. Migration examples match the built CLI. ## Stage 9: Verify The Complete Native Cutover +**Status: Complete.** Verified the assembled native CLI on Go 1.27.4 with +`go test ./... -count=1`, `go test -race ./... -count=1`, `go vet ./...`, +and `go build ./cmd/scriptorium`. Formatting, diff whitespace, module tidiness, +shell syntax, maintained examples, and local Markdown links/anchors pass. +The maintained discovery configuration now has an end-to-end controlled-provider +test covering support/review inventories, requirements before artifacts exist, +concrete validation, preview/run context agreement, and independent native app +and framework comparisons. Existing checkout tests verify both optional entry +spellings and discovery on later invocations. Additional checks cover rejected +legacy surfaces, partial output, diagnostic delivery failure, and real child +process interrupt/SIGTERM cancellation reaching a provider request. + +Built Linux amd64 and arm64 with Go 1.27.4, `CGO_ENABLED=0`, `-trimpath`, +and `-s -w` plus version/revision linker metadata. The amd64 binary reports +the injected `verification` identity and full source revision +`bd7136ceebbdc7498e9a768ce8abbe0cbb00318f`; arm64 was cross-built, not executed. +The binary also exposes help, native scope discovery, and Bash/Zsh/Fish +completion without loading configuration or contacting a provider. + +Selected releases remain CommandKit v0.7.0 +(`b57bfe176cff05a892e6cb07ebb5f350239e55d4`), ckconfig v0.9.0 +(`9962368f0c1ea1f6ad010638e94621091105d745`), ckprompt v0.10.0 +(`c9a56e1b15f2169881a91fca4e1f4a1c475e057a`), ConfigKit v1.2.1 +(`51ef3144158db1a09fabdbd1c7ae59908c84923b`), and PromptKit v0.23.3 +(`804a4d32b23aa2e7973400bb0a322a7016574c9f`). Module and linked-runtime +inspection confirms no replacements, workspace, vendoring, or linked maintainer +tools; attribution follows the linked dependencies, including conditional +Windows mousetrap. Public native adapters, request bindings, reporters, +`RenderReport`, and typed outcomes remain the selected integration APIs. + +The final implementation audit against this plan and ADRs 0004–0006 found no +remaining required work. HTTP/report implementation and accepted ADRs remain +unchanged; no compatibility surface, post-resolution config rewriting, extra +readiness gate, workflow state, snapshot promise, or transactional delivery was +added. Embedded framework configuration, dynamic-resource completion, the +optional format accessor, and HTTP redesign remain outside this migration. +No release was published and no commits were pushed. + 1. Finish any deferred stage checks against the complete application. Use real ConfigKit, PromptKit, and CommandKit with deterministic provider substitutes at the external boundary. Tests remain offline and independent of real keys. diff --git a/docs/roadmap/narratio-feature-request.md b/docs/roadmap/narratio-feature-request.md index 67174fe..942104c 100644 --- a/docs/roadmap/narratio-feature-request.md +++ b/docs/roadmap/narratio-feature-request.md @@ -1,7 +1,8 @@ # Prompt input requirements for Narratio -Status: the native interface and focused consumer scenarios are implemented. -The [implementation plan](implementation.md) tracks final assembled verification. +Status: the native interface and consumer acceptance scenarios are implemented +and verified, including the assembled collection workflow. The +[implementation plan](implementation.md) records the verification evidence. This proposal records Narratio's consumer needs and their mapping to the native interface; the [CLI contract](../cli.md) owns current behavior. The [CommandKit notepad](commandkit-notepad.md) records the migration decisions. diff --git a/internal/adapter/cli/delivery_test.go b/internal/adapter/cli/delivery_test.go index a5c922e..e40d82a 100644 --- a/internal/adapter/cli/delivery_test.go +++ b/internal/adapter/cli/delivery_test.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "net/http" "net/http/httptest" "os" @@ -129,6 +130,41 @@ type failedDeliveryWriter struct{ cause error } func (writer failedDeliveryWriter) Write([]byte) (int, error) { return 0, writer.cause } +type partialDeliveryWriter struct { + bytes.Buffer + cause error +} + +func (writer *partialDeliveryWriter) Write(value []byte) (int, error) { + count := min(2, len(value)) + _, _ = writer.Buffer.Write(value[:count]) + return count, writer.cause +} + +func TestPartialOutputAndDiagnosticFailureUseNativeFinalization(t *testing.T) { + t.Chdir(t.TempDir()) + failure := errors.New("presentation failure") + app, err := commandkit.New(commandkit.Identity{Name: "delivery"}, commandkit.NewCommand(commandkit.CommandSpec[struct{}]{Name: "run", Run: func(inv commandkit.Invocation, _ struct{}) commandkit.Outcome { + return deliverRunResult(inv, runInput{ReportPath: commandkit.Supplied[string]{Set: true, Value: "report"}}, &promptkit.RunResult{Artifact: promptkit.Artifact{Body: []byte("artifact")}}) + }})) + if err != nil { + t.Fatal(err) + } + output := &partialDeliveryWriter{cause: failure} + var diagnostics bytes.Buffer + outcome := app.Execute(context.Background(), []string{"run"}, commandkit.Streams{Out: output, Err: &diagnostics}) + if outcome.Status != 1 || output.String() != "ar" || !errors.Is(outcome.Presentation.Output.WriteError, failure) { + t.Fatalf("partial output was lost: %+v %q", outcome, output.String()) + } + if _, err := os.Stat("report"); err != nil { + t.Fatal("report was not attempted", err) + } + outcome = Execute(context.Background(), commandkit.Identity{Name: "scriptorium"}, []string{"run"}, io.Discard, failedDeliveryWriter{failure}) + if outcome.Status != 1 || outcome.Kind != commandkit.OutcomeOperational || !errors.Is(outcome.Presentation.Diagnostics.WriteError, failure) { + t.Fatalf("usage diagnostic failure not finalized: %+v", outcome) + } +} + func TestNativeDeliveryAttemptsReportAfterStdoutFailure(t *testing.T) { t.Chdir(t.TempDir()) result := &promptkit.RunResult{RunID: "known-result", Artifact: promptkit.Artifact{Body: []byte("body")}} diff --git a/internal/adapter/cli/discovery_test.go b/internal/adapter/cli/discovery_test.go index d8f6ddd..c87a1b2 100644 --- a/internal/adapter/cli/discovery_test.go +++ b/internal/adapter/cli/discovery_test.go @@ -3,11 +3,15 @@ package cli import ( "encoding/json" "fmt" - "go.maximumdirect.net/promptkit" + "net/http" + "net/http/httptest" "os" "path/filepath" + "reflect" "strings" "testing" + + "go.maximumdirect.net/promptkit" ) func TestCollectionCheckoutDiscoveryAcrossInvocations(t *testing.T) { @@ -91,3 +95,102 @@ func TestCollectionCheckoutDiscoveryAcrossInvocations(t *testing.T) { }) } } + +func TestMaintainedCollectionWorkflow(t *testing.T) { + root, err := filepath.Abs(filepath.Join("..", "..", "..")) + if err != nil { + t.Fatal(err) + } + t.Chdir(root) + requests := make(chan []promptkit.RenderedMessage, 2) + provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + var body struct { + Messages []promptkit.RenderedMessage `json:"messages"` + } + if err := json.NewDecoder(request.Body).Decode(&body); err != nil { + t.Error(err) + } + select { + case requests <- body.Messages: + default: + t.Error("unexpected provider dispatch") + } + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"choices":[{"message":{"content":"collection result"}}]}`) + })) + defer provider.Close() + const config = "examples/config.discovery.yml" + outcome, inventory, diagnostics := executeNative(t, "promptkit", "list", "prompts", "--all-namespaces", "--config="+config, "--format=json") + if outcome.Status != 0 || !strings.Contains(inventory, "support/answer") || !strings.Contains(inventory, "support/summarize") || !strings.Contains(inventory, "review/answer") { + t.Fatalf("collection discovery: %+v %q %q", outcome, inventory, diagnostics) + } + inputPath := filepath.Join(t.TempDir(), "future-transcript.md") + for _, namespace := range []string{"support", "review"} { + profile := "fast" + if namespace == "review" { + profile = "base" + } + outcome, output, diagnostics := executeNative(t, "promptkit", "requirements", "show", "answer", "--config="+config, "--prompt-namespace="+namespace, "--profile="+profile, "--profile-namespace="+namespace, "--format=json") + var report consumerRequirements + if err := json.Unmarshal([]byte(output), &report); err != nil { + t.Fatal(err) + } + if outcome.Status != 0 || !report.Complete || report.PromptID != namespace+"/answer" || report.ProfileID != namespace+"/"+profile { + t.Fatalf("requirements: %+v %+v %q", outcome, report, diagnostics) + } + if namespace == "support" && (len(report.Inputs) != 1 || report.Inputs[0].Name != "transcript" || report.Inputs[0].Kind != promptkit.RequirementRequired) { + t.Fatalf("required binding: %+v", report) + } + } + if _, err := os.Stat(inputPath); !os.IsNotExist(err) { + t.Fatalf("discovery created input artifact: %v", err) + } + writeNativeFile(t, inputPath, "generated collection transcript") + for _, namespace := range []string{"support", "review"} { + profile := "fast" + if namespace == "review" { + profile = "base" + } + flags := []string{"answer", "--config=" + config, "--prompt-namespace=" + namespace, "--profile=" + profile, "--profile-namespace=" + namespace, "--endpoint=" + provider.URL + "/v1", "--model=controlled", "--timeout-seconds=0", "--transient-retries=0", "--session-id=collection-session", "--format=json"} + if namespace == "support" { + flags = append(flags, "--input-file=transcript="+inputPath) + } + outcome, _, diagnostics := executeNative(t, append([]string{"promptkit", "requirements", "validate"}, flags...)...) + if outcome.Status != 0 { + t.Fatalf("validation: %+v %q", outcome, diagnostics) + } + outcome, output, diagnostics := executeNative(t, append([]string{"promptkit", "show", "prepared"}, flags...)...) + var prepared promptkit.PreparedRun + if err := json.Unmarshal([]byte(output), &prepared); err != nil { + t.Fatal(err) + } + if outcome.Status != 0 { + t.Fatalf("preview: %+v %q", outcome, diagnostics) + } + outcome, output, diagnostics = executeNative(t, append([]string{"run"}, flags...)...) + var result promptkit.RunResult + if err := json.Unmarshal([]byte(output), &result); err != nil { + t.Fatal(err) + } + if outcome.Status != 0 || result.PromptID != prepared.PromptID || result.SelectedProfileID != prepared.SelectedProfileID || result.RenderedPromptHash != prepared.RenderedPromptHash || !reflect.DeepEqual(result.InputHashes, prepared.InputHashes) || string(result.Artifact.Body) != "collection result" { + t.Fatalf("workflow selection/context diverged: prepared=%+v result=%+v outcome=%+v %q", prepared, result, outcome, diagnostics) + } + select { + case messages := <-requests: + if !reflect.DeepEqual(messages, prepared.Messages) { + t.Fatalf("provider saw different messages: %+v %+v", messages, prepared.Messages) + } + default: + t.Fatal("generation did not reach controlled provider") + } + } + for _, args := range [][]string{ + {"config", "diff", "--scope=app", "--config=" + config, "--against-config=" + config, `--set=/promptkit/schema_dir="left"`, `--against-set=/promptkit/schema_dir="right"`, "--check"}, + {"config", "diff", "--scope=promptkit", "--config=" + config, "--against-config=examples/config.yml", "--check"}, + } { + outcome, output, diagnostics := executeNative(t, args...) + if outcome.Status != 3 || output == "" { + t.Fatalf("independent configuration sides: %+v %q %q", outcome, output, diagnostics) + } + } +} diff --git a/internal/adapter/cli/native_run_test.go b/internal/adapter/cli/native_run_test.go index 42e160f..4cd0a43 100644 --- a/internal/adapter/cli/native_run_test.go +++ b/internal/adapter/cli/native_run_test.go @@ -168,10 +168,13 @@ func TestRunDistinguishesCompletedInvalidContentAndOperationalFailure(t *testing func TestRunUsageAndConfigurationFailures(t *testing.T) { t.Chdir(t.TempDir()) for _, args := range [][]string{ + {"render"}, {"inspect", "prompt"}, {"run"}, {"run", "one", "two"}, {"run", "one", "--prompt=two"}, {"run", "one", "--llm-prompt=two"}, {"run", "one", "--backend=local"}, {"run", "one", "--out=result"}, {"run", "one", "--execution-report=result"}, {"run", "one", "--llm-base-url=http://example.invalid"}, {"run", "one", "--timeout=1s"}, {"run", "one", "--prompt-id=two"}, {"run", "one", "--profile-id=local"}, {"run", "one", "--run-timeout=bad"}, + {"run", "one", "--prompt-version=1"}, {"run", "one", "--prompt-dir=prompts"}, {"run", "one", "--profile-dir=profiles"}, + {"run", "one", "--config-overlay=local"}, {"run", "one", "--output=file"}, } { outcome, _, _ := executeNative(t, args...) if outcome.Kind != commandkit.OutcomeUsage || outcome.Status != 2 {