diff --git a/scripts/test-release.py b/scripts/test-release.py index 349d750..8e60f80 100644 --- a/scripts/test-release.py +++ b/scripts/test-release.py @@ -1,4 +1,8 @@ -"""Exercise ReleaseKit publication wiring in disposable repositories.""" +"""Check Promptkit's launcher and release profile using a disposable local remote. + +ReleaseKit owns the detailed publication, validation, and delivery algorithms. +These fixtures protect this repository's tool selection and product-check wiring. +""" import json import os @@ -11,495 +15,163 @@ from pathlib import Path REPOSITORY_ROOT = Path(__file__).resolve().parent.parent -RELEASE_VERSION = "v0.0.1" -FAILED_RELEASE_VERSION = "v0.0.2" RELEASEKIT_MODULE = "go.maximumdirect.net/releasekit" +RELEASE_VERSION = "v0.0.1" +FAILED_VERSION = "v0.0.2" -def execute(command, cwd, environment): - try: - return subprocess.run( - command, - cwd=cwd, - env=environment, - text=True, - capture_output=True, - check=False, +def run(command, directory, environment, status=0): + result = subprocess.run( + command, cwd=directory, env=environment, text=True, capture_output=True + ) + if result.returncode != status: + raise RuntimeError( + f"{' '.join(map(str, command))}: status {result.returncode}, want {status}" + f"\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}" ) - except OSError as error: - raise RuntimeError(f"could not run {command[0]}: {error}") from error - - -def run(command, cwd, environment): - result = execute(command, cwd, environment) - if result.returncode != 0: - details = [ - f"command failed with status {result.returncode}: {' '.join(command)}" - ] - if result.stdout: - details.append(f"stdout:\n{result.stdout.rstrip()}") - if result.stderr: - details.append(f"stderr:\n{result.stderr.rstrip()}") - raise RuntimeError("\n".join(details)) return result.stdout -def write_file(path, contents): +def git(repository, environment, *arguments): + return run(["git", "-C", str(repository), *arguments], repository.parent, environment).strip() + + +def write(path, contents): path.parent.mkdir(parents=True, exist_ok=True) path.write_text(textwrap.dedent(contents).lstrip("\n"), encoding="utf-8") -def repo_git(repository, environment, *arguments): - return run( - ["git", "-C", str(repository), *arguments], - cwd=repository.parent, - environment=environment, - ) +def prepare_fixture(root, environment, module_bytes): + repository, remote = root / "checkout", root / "origin.git" + run(["git", "init", "--bare", "--initial-branch=main", str(remote)], root, environment) + run(["git", "init", "--initial-branch=main", str(repository)], root, environment) + git(repository, environment, "config", "user.name", "Promptkit Fixture") + git(repository, environment, "config", "user.email", "fixture@example.invalid") + git(repository, environment, "remote", "add", "origin", str(remote)) + for name, contents in module_bytes.items(): + (repository / name).write_bytes(contents) + for name in ("release.sh", "releasekit.sh"): + destination = repository / "scripts" / name + destination.parent.mkdir(exist_ok=True) + shutil.copy2(REPOSITORY_ROOT / "scripts" / name, destination) + shutil.copy2(REPOSITORY_ROOT / "releasekit.json", repository / "releasekit.json") + shutil.copy2(REPOSITORY_ROOT / ".gitignore", repository / ".gitignore") + for version in (RELEASE_VERSION, FAILED_VERSION): + write(repository / f"docs/releases/{version}.md", f"# Promptkit {version}\n\nFixture.\n") + write(repository / "scripts/check-examples.py", """ + import os + import sys - -def remote_git(remote, temporary_root, environment, *arguments): - return run( - ["git", "--git-dir", str(remote), *arguments], - cwd=temporary_root, - environment=environment, - ) - - -def fixture_sources(repository, environment): - write_file( - repository / "fixture.go", - """ - package fixture - - func candidateValue() string { return "ready" } - """, - ) - write_file( - repository / "fixture_test.go", - f""" + with open(os.environ["FIXTURE_CHECK_MARKER"], "a") as marker: + marker.write("called\\n") + if os.environ.get("FIXTURE_CHECK_FAILURE"): + print("fixture example check rejected candidate", file=sys.stderr) + sys.exit(1) + """) + write(repository / "fixture_test.go", """ package fixture import ( - "encoding/json" + "fmt" "os" "runtime" "testing" ) - func TestCandidateSuite(t *testing.T) {{ - if candidateValue() != "ready" {{ - t.Fatal("candidate package is not ready") - }} - marker := os.Getenv("FIXTURE_SUITE_MARKER") - if marker == "" {{ - t.Fatal("FIXTURE_SUITE_MARKER is not set") - }} - record := struct {{ - GoVersion string `json:"go_version"` - Race bool `json:"race"` - }}{{GoVersion: runtime.Version(), Race: raceDetectorEnabled}} - data, err := json.Marshal(record) - if err != nil {{ - t.Fatal(err) - }} - file, err := os.OpenFile(marker, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) - if err != nil {{ - t.Fatal(err) - }} - if _, err := file.Write(append(data, '\\n')); err != nil {{ - file.Close() - t.Fatal(err) - }} - if err := file.Close(); err != nil {{ - t.Fatal(err) - }} - }} - """, - ) - write_file( - repository / "race_enabled_test.go", - """ - //go:build race - - package fixture - - const raceDetectorEnabled = true - """, - ) - write_file( - repository / "race_disabled_test.go", - """ - //go:build !race - - package fixture - - const raceDetectorEnabled = false - """, - ) - go_files = [ - str(repository / filename) - for filename in ( - "fixture.go", - "fixture_test.go", - "race_enabled_test.go", - "race_disabled_test.go", - ) - ] - run(["gofmt", "-w", *go_files], cwd=repository, environment=environment) - - -def prepare_repository(repository, remote, environment, module_bytes, version): - run( - ["git", "init", "--bare", "--initial-branch=main", str(remote)], - cwd=repository.parent, - environment=environment, - ) - run( - ["git", "init", "--initial-branch=main", str(repository)], - cwd=repository.parent, - environment=environment, - ) - repo_git(repository, environment, "config", "user.name", "ReleaseKit Fixture") - repo_git(repository, environment, "config", "user.email", "fixture@example.invalid") - repo_git(repository, environment, "remote", "add", "origin", str(remote)) - - for name, contents in module_bytes.items(): - (repository / name).write_bytes(contents) - - write_file( - repository / ".gitignore", - """ - .releasekit/ - """, - ) - write_file( - repository / "releasekit.json", - (REPOSITORY_ROOT / "releasekit.json").read_text(encoding="utf-8"), - ) - - scripts = repository / "scripts" - scripts.mkdir(parents=True, exist_ok=True) - shutil.copy2(REPOSITORY_ROOT / "scripts/release.sh", scripts / "release.sh") - shutil.copy2(REPOSITORY_ROOT / "scripts/releasekit.sh", scripts / "releasekit.sh") - write_file( - scripts / "check-examples.py", - """ - import os - from pathlib import Path - import sys - - marker = Path(os.environ["FIXTURE_EXAMPLE_MARKER"]) - with marker.open("a", encoding="utf-8") as stream: - stream.write("called\\n") - if os.environ.get("FIXTURE_EXAMPLE_FAILURE") == "1": - print("fixture example check rejected candidate", file=sys.stderr) - raise SystemExit(1) - """, - ) - write_file( - repository / f"docs/releases/{version}.md", - f""" - # Promptkit {version} - - Disposable publication fixture. - """, - ) - fixture_sources(repository, environment) - - repo_git(repository, environment, "add", "-A") - run( - ["git", "-C", str(repository), "commit", "-m", "Add disposable release fixture"], - cwd=repository.parent, - environment=environment, - ) - repo_git(repository, environment, "push", "origin", "main") - - -def verify_publication(repository, remote, temporary_root, environment, module_bytes, selected_go): - profile = json.loads((repository / "releasekit.json").read_text(encoding="utf-8")) - token_path = repository / profile["delivery"]["token_file"] - if token_path.exists() or environment.get("RELEASEKIT_GITEA_TOKEN"): - raise RuntimeError("publication fixture must not have delivery credentials") - selected_releasekit = run( - ["go", "list", "-mod=readonly", "-m", "-f", "{{.Version}}", RELEASEKIT_MODULE], - cwd=repository, - environment=environment, - ).strip() - publication = repository / "scripts/release.sh" - run( - [str(publication), RELEASE_VERSION], - cwd=temporary_root, - environment=environment, - ) - - suite_lines = Path(environment["FIXTURE_SUITE_MARKER"]).read_text( - encoding="utf-8" - ).splitlines() - if len(suite_lines) != 1: - raise RuntimeError(f"expected one suite marker, found {len(suite_lines)}") - suite_record = json.loads(suite_lines[0]) - if suite_record != {"go_version": selected_go, "race": True}: - raise RuntimeError(f"unexpected suite marker: {suite_record!r}") - - example_lines = Path(environment["FIXTURE_EXAMPLE_MARKER"]).read_text( - encoding="utf-8" - ).splitlines() - if example_lines != ["called"]: - raise RuntimeError(f"expected one additional-check marker, got {example_lines!r}") - - binary = repository / ".releasekit/bin/releasekit" - if not binary.is_file(): - raise RuntimeError("ReleaseKit bootstrap did not leave its ignored executable") - build_info = run( - ["go", "version", "-m", str(binary)], - cwd=temporary_root, - environment=environment, - ) - first_line = build_info.splitlines()[0] - if selected_go not in first_line: - raise RuntimeError(f"tool was built with {first_line!r}, expected {selected_go}") - expected_module = f"\tmod\t{RELEASEKIT_MODULE}\t{selected_releasekit}\t" - if expected_module not in build_info: - raise RuntimeError( - f"bootstrap executable does not record selected ReleaseKit {selected_releasekit}" - ) - - candidate = repo_git(repository, environment, "rev-parse", "HEAD").strip() - if repo_git(repository, environment, "rev-parse", "main").strip() != candidate: - raise RuntimeError("fixture main moved away from the published candidate") - if remote_git(remote, temporary_root, environment, "rev-parse", "refs/heads/main").strip() != candidate: - raise RuntimeError("local origin main does not identify the fixture candidate") - - tag_ref = f"refs/tags/{RELEASE_VERSION}" - if repo_git(repository, environment, "cat-file", "-t", tag_ref).strip() != "tag": - raise RuntimeError("local release ref is not an annotated tag") - local_tag = repo_git(repository, environment, "rev-parse", tag_ref).strip() - local_tag_commit = repo_git( - repository, environment, "rev-parse", f"{tag_ref}^{{}}" - ).strip() - if local_tag_commit != candidate: - raise RuntimeError("local annotated tag does not identify the fixture candidate") - - remote_tag = remote_git(remote, temporary_root, environment, "rev-parse", tag_ref).strip() - remote_tag_commit = remote_git( - remote, temporary_root, environment, "rev-parse", f"{tag_ref}^{{}}" - ).strip() - if remote_tag != local_tag or remote_tag_commit != candidate: - raise RuntimeError("remote annotated tag does not identify the fixture candidate") - if remote_git( - remote, temporary_root, environment, "cat-file", "-t", tag_ref - ).strip() != "tag": - raise RuntimeError("remote release ref is not an annotated tag") - - tag_object = repo_git(repository, environment, "cat-file", "tag", tag_ref) - separator = tag_object.find("\n\n") - if separator < 0: - raise RuntimeError("annotated tag has no message") - tag_message = tag_object[separator + 2 :] - note = (repository / f"docs/releases/{RELEASE_VERSION}.md").read_text( - encoding="utf-8" - ) - if tag_message != note: - raise RuntimeError("annotated tag message does not match the committed release note") - - for name, contents in module_bytes.items(): - if (repository / name).read_bytes() != contents: - raise RuntimeError(f"publication changed {name}") - - status = repo_git( - repository, environment, "status", "--porcelain", "--untracked-files=all" - ) - if status: - raise RuntimeError(f"fixture checkout is not clean after publication:\n{status}") - repo_git(repository, environment, "check-ignore", "-q", ".releasekit/bin/releasekit") - - -def verify_validation_failure( - repository, remote, temporary_root, environment, module_bytes, selected_go -): - scenario_environment = environment.copy() - suite_marker = temporary_root / "failure-suite.jsonl" - example_marker = temporary_root / "failure-example.log" - scenario_environment.update( - { - "FIXTURE_SUITE_MARKER": str(suite_marker), - "FIXTURE_EXAMPLE_MARKER": str(example_marker), - "FIXTURE_EXAMPLE_FAILURE": "1", + func TestSelectedSuite(t *testing.T) { + if !raceEnabled { + t.Fatal("profile did not select the race detector") + } + file, err := os.OpenFile(os.Getenv("FIXTURE_SUITE_MARKER"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) + if err != nil { t.Fatal(err) } + defer file.Close() + if _, err := fmt.Fprintln(file, runtime.Version()); err != nil { t.Fatal(err) } } + """) + for name, tag, enabled in (("race", "race", "true"), ("norace", "!race", "false")): + write(repository / f"{name}.go", f"//go:build {tag}\n\npackage fixture\n\nconst raceEnabled = {enabled}\n") + run(["gofmt", "-w", "fixture_test.go", "race.go", "norace.go"], repository, environment) + git(repository, environment, "add", ".") + git(repository, environment, "commit", "-m", "Add release wiring fixture") + git(repository, environment, "push", "origin", "main") + return repository, remote + + +def verify_bootstrap(repository, root, environment, selected_go): + selected_version = run( + ["go", "list", "-mod=readonly", "-m", "-f", "{{.Version}}", RELEASEKIT_MODULE], + repository, environment, + ).strip() + launcher = str(repository / "scripts/releasekit.sh") + version = run([launcher, "version"], root, environment).strip() + if version != f"releasekit {selected_version}": + raise RuntimeError(f"launcher used an unexpected tool: {version}") + build_info = run( + ["go", "version", "-m", str(repository / ".releasekit/bin/releasekit")], + root, environment, ) - prepare_repository( - repository, - remote, - scenario_environment, - module_bytes, - FAILED_RELEASE_VERSION, + if selected_go not in build_info.splitlines()[0]: + raise RuntimeError("launcher did not build with the selected local Go") + if f"\tmod\t{RELEASEKIT_MODULE}\t{selected_version}\t" not in build_info: + raise RuntimeError("launcher did not use the module-selected ReleaseKit") + run([launcher, "unknown"], root, environment, status=2) + + +def verify_publication(repository, remote, root, environment, profile, selected_go): + if (repository / profile["delivery"]["token_file"]).exists(): + raise RuntimeError("source-publication fixture must have no delivery token") + publication = str(repository / "scripts/release.sh") + run([publication, RELEASE_VERSION], root, environment) + candidate = git(repository, environment, "rev-parse", "HEAD") + if git(remote, environment, "rev-parse", f"refs/tags/{RELEASE_VERSION}^{{}}") != candidate: + raise RuntimeError("launcher did not publish the intended candidate") + suite = Path(environment["FIXTURE_SUITE_MARKER"]).read_text().splitlines() + checks = Path(environment["FIXTURE_CHECK_MARKER"]).read_text().splitlines() + if suite != [selected_go] or checks != ["called"]: + raise RuntimeError(f"profile did not run the selected suite/check: {suite}, {checks}") + + failed_environment = dict(environment, FIXTURE_CHECK_FAILURE="1") + run([publication, FAILED_VERSION], root, failed_environment, status=1) + if Path(environment["FIXTURE_CHECK_MARKER"]).read_text().splitlines() != ["called", "called"]: + raise RuntimeError("failure scenario did not exercise the product check") + run( + ["git", "-C", str(remote), "show-ref", "--verify", "--quiet", f"refs/tags/{FAILED_VERSION}"], + root, environment, status=1, ) - result = execute( - [str(repository / "scripts/release.sh"), FAILED_RELEASE_VERSION], - cwd=temporary_root, - environment=scenario_environment, - ) - if result.returncode != 1: - raise RuntimeError( - "validation failure returned status " - f"{result.returncode}, want 1; stdout:\n{result.stdout}\nstderr:\n{result.stderr}" - ) - if "fixture example check rejected candidate" not in result.stderr: - raise RuntimeError(f"failure diagnostic omitted useful context:\n{result.stderr}") - suite_lines = suite_marker.read_text(encoding="utf-8").splitlines() - if len(suite_lines) != 1: - raise RuntimeError(f"expected one failure-case suite marker, found {len(suite_lines)}") - suite_record = json.loads(suite_lines[0]) - if suite_record != {"go_version": selected_go, "race": True}: - raise RuntimeError(f"unexpected failure-case suite marker: {suite_record!r}") - if example_marker.read_text(encoding="utf-8").splitlines() != ["called"]: - raise RuntimeError("expected one additional-check invocation before failure") - - tag_ref = f"refs/tags/{FAILED_RELEASE_VERSION}" - for description, command in ( - ( - "local", - ["git", "-C", str(repository), "show-ref", "--verify", "--quiet", tag_ref], - ), - ( - "remote", - ["git", "--git-dir", str(remote), "show-ref", "--verify", "--quiet", tag_ref], - ), - ): - result = execute(command, cwd=temporary_root, environment=scenario_environment) - if result.returncode == 0: - raise RuntimeError(f"validation failure created the {description} tag {tag_ref}") - if result.returncode != 1: - raise RuntimeError( - f"could not verify {description} tag absence: " - f"status {result.returncode}: {result.stderr.strip()}" - ) - - for name, contents in module_bytes.items(): - if (repository / name).read_bytes() != contents: - raise RuntimeError(f"failed publication changed {name}") - status = repo_git( - repository, - scenario_environment, - "status", - "--porcelain", - "--untracked-files=all", - ) - if status: - raise RuntimeError(f"failure fixture checkout is not clean:\n{status}") - - -def verify_forbidden_token(repository, environment): - profile = json.loads((repository / "releasekit.json").read_text(encoding="utf-8")) +def verify_token_guard(repository, environment, profile): token_path = profile["delivery"]["token_file"] - write_file(repository / token_path, "disposable fixture placeholder\n") - note_path = f"docs/releases/{FAILED_RELEASE_VERSION}.md" - write_file(repository / note_path, f"# Promptkit {FAILED_RELEASE_VERSION}\n\nFixture.\n") - repo_git(repository, environment, "add", "--", note_path) - repo_git(repository, environment, "add", "-f", "--", token_path) - repo_git(repository, environment, "commit", "-m", "Add forbidden-path fixture") - - result = execute( - [str(repository / ".releasekit/bin/releasekit"), "check", FAILED_RELEASE_VERSION], - cwd=repository, - environment=environment, - ) - if ( - result.returncode != 1 - or "forbidden-files" not in result.stderr - or token_path not in result.stderr - ): - raise RuntimeError( - f"committed delivery token path was not rejected:\n{result.stdout}\n{result.stderr}" - ) - - -def verify_usage_classification(binary, temporary_root, environment): - result = execute( - [str(binary), "unknown"], cwd=temporary_root, environment=environment - ) - if result.returncode != 2: - raise RuntimeError( - f"unsupported command returned status {result.returncode}, want 2; " - f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}" - ) - if result.stdout: - raise RuntimeError(f"unsupported command wrote to stdout: {result.stdout!r}") - if 'unknown command "unknown"' not in result.stderr or "usage: releasekit" not in result.stderr: - raise RuntimeError(f"unsupported command diagnostic was not useful:\n{result.stderr}") + git(repository, environment, "check-ignore", "--no-index", token_path) + if token_path not in profile["validation"].get("forbidden_files", []): + raise RuntimeError("delivery token path is not selected for committed-file protection") def main(): - environment = os.environ.copy() - environment.pop("RELEASEKIT_GITEA_TOKEN", None) - environment.update( - { - "GOTOOLCHAIN": "local", - "GOWORK": "off", - "GOPROXY": "off", - "GONOPROXY": "none", - "GOSUMDB": "off", - } - ) - selected_go = run( - ["go", "env", "GOVERSION"], - cwd=REPOSITORY_ROOT, - environment=environment, - ).strip() - if not selected_go: - print("go env GOVERSION returned no version", file=sys.stderr) + environment = dict(os.environ, GOTOOLCHAIN="local", GOWORK="off", + GOPROXY="off", GONOPROXY="none", GOSUMDB="off") + profile = json.loads((REPOSITORY_ROOT / "releasekit.json").read_text()) + environment.pop(profile["delivery"].get("token_env", "RELEASEKIT_GITEA_TOKEN"), None) + environment.pop("FIXTURE_CHECK_FAILURE", None) + module_bytes = {name: (REPOSITORY_ROOT / name).read_bytes() for name in ("go.mod", "go.sum")} + try: + selected_go = run(["go", "env", "GOVERSION"], REPOSITORY_ROOT, environment).strip() + with tempfile.TemporaryDirectory(prefix="promptkit-release-fixture-") as directory: + root = Path(directory) + environment.update(FIXTURE_CHECK_MARKER=str(root / "checks"), + FIXTURE_SUITE_MARKER=str(root / "suite")) + repository, remote = prepare_fixture(root, environment, module_bytes) + verify_bootstrap(repository, root, environment, selected_go) + verify_publication(repository, remote, root, environment, profile, selected_go) + verify_token_guard(repository, environment, profile) + if any((repository / name).read_bytes() != contents for name, contents in module_bytes.items()): + raise RuntimeError("release commands changed module metadata") + if git(repository, environment, "status", "--porcelain", "--untracked-files=all"): + raise RuntimeError("release commands left unexpected worktree changes") + except (OSError, RuntimeError) as error: + print(error, file=sys.stderr) return 1 - - ignored_rule = ".releasekit/" - ignore_lines = (REPOSITORY_ROOT / ".gitignore").read_text( - encoding="utf-8" - ).splitlines() - if ignored_rule not in ignore_lines: - print(".gitignore does not contain the ReleaseKit output rule", file=sys.stderr) - return 1 - - module_bytes = { - name: (REPOSITORY_ROOT / name).read_bytes() for name in ("go.mod", "go.sum") - } - with tempfile.TemporaryDirectory(prefix="promptkit-release-fixture-") as directory: - temporary_root = Path(directory) - repository = temporary_root / "checkout" - remote = temporary_root / "origin.git" - suite_marker = temporary_root / "suite.jsonl" - example_marker = temporary_root / "example.log" - environment["FIXTURE_SUITE_MARKER"] = str(suite_marker) - environment["FIXTURE_EXAMPLE_MARKER"] = str(example_marker) - try: - prepare_repository( - repository, remote, environment, module_bytes, RELEASE_VERSION - ) - verify_publication( - repository, - remote, - temporary_root, - environment, - module_bytes, - selected_go, - ) - verify_usage_classification( - repository / ".releasekit/bin/releasekit", temporary_root, environment - ) - verify_validation_failure( - temporary_root / "failure-checkout", - temporary_root / "failure-origin.git", - temporary_root, - environment, - module_bytes, - selected_go, - ) - verify_forbidden_token(repository, environment) - except (RuntimeError, OSError, json.JSONDecodeError) as error: - print(error, file=sys.stderr) - return 1 - - print("ReleaseKit publication and committed-token-path fixtures passed") + print("Promptkit release wiring checks passed") return 0