diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index f0a91d2..1d80659 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -185,6 +185,12 @@ Validation: run `python3 scripts/test-release.py` using the selected Go on repositories. Check the profile diff to confirm only the obsolete custom hook was removed. Do not test by publishing a real tag or hosted page. +Completion: removed the Python example runner and its custom profile check; +the release-wiring diagnostic now tests the native missing-note failure and +preserves its other launcher, suite, publication, and token-path checks. The +diagnostic passed with Go 1.27.0 against disposable local repositories, and the +profile diff retains the existing built-in validators and delivery settings. + ## Stage 5: Align Documentation And Verify The Complete Sprint Scope: the development guide, local testing/release policies, release runbook, diff --git a/releasekit.json b/releasekit.json index b87925a..5e7660c 100644 --- a/releasekit.json +++ b/releasekit.json @@ -20,18 +20,5 @@ "owner": "base-layer", "repository": "promptkit", "token_file": ".releasekit/gitea-token" - }, - "checks": [ - { - "args": ["python3", "./scripts/check-examples.py"], - "env": { - "GOTOOLCHAIN": "local", - "GOWORK": "off", - "GOFLAGS": "-mod=readonly", - "GOPROXY": "off", - "GONOPROXY": "none", - "GOSUMDB": "off" - } - } - ] + } } diff --git a/scripts/check-examples.py b/scripts/check-examples.py deleted file mode 100644 index 841bc31..0000000 --- a/scripts/check-examples.py +++ /dev/null @@ -1,181 +0,0 @@ -#!/usr/bin/env python3 -"""Run the maintained consumer examples and check their semantic output.""" - -import json -import os -import subprocess -import sys -from pathlib import Path - - -REPOSITORY_ROOT = Path(__file__).resolve().parent.parent - - -def example_environment(): - environment = os.environ.copy() - environment["GOTOOLCHAIN"] = "local" - try: - host = subprocess.run( - ["go", "env", "GOHOSTOS", "GOHOSTARCH"], - cwd=REPOSITORY_ROOT, - env=environment, - text=True, - capture_output=True, - check=False, - ) - except OSError as error: - raise RuntimeError(f"could not run go env: {error}") from error - - if host.returncode != 0: - raise RuntimeError( - "go env GOHOSTOS GOHOSTARCH failed " - f"with exit status {host.returncode}: {host.stderr.strip()}" - ) - - values = host.stdout.splitlines() - if len(values) != 2 or not all(values): - raise RuntimeError( - "go env GOHOSTOS GOHOSTARCH returned unexpected output: " - f"{host.stdout!r}" - ) - - environment["GOOS"], environment["GOARCH"] = values - return environment - - -def run_example(name, environment, failures): - command = ["go", "run", f"./examples/go-library/{name}"] - try: - result = subprocess.run( - command, - cwd=REPOSITORY_ROOT, - env=environment, - text=True, - capture_output=True, - check=False, - ) - except OSError as error: - failures.append(f"{name}: could not start {command[0]}: {error}") - return None - - if result.returncode != 0: - failures.append(f"{name}: command exited with status {result.returncode}") - if result.stderr: - failures.append(f"{name}: stderr:\n{result.stderr.rstrip()}") - if result.stdout: - failures.append(f"{name}: stdout:\n{result.stdout.rstrip()}") - return None - - try: - value = json.loads(result.stdout) - except json.JSONDecodeError as error: - failures.append(f"{name}: output is not valid JSON: {error}") - if result.stdout: - failures.append(f"{name}: stdout:\n{result.stdout.rstrip()}") - return None - - if not isinstance(value, dict): - failures.append( - f"{name}: expected a JSON object, got {type(value).__name__}" - ) - return None - - return value - - -def expect_equal(example_name, value, key, expected, failures): - actual = value.get(key) - matches = actual is expected if isinstance(expected, bool) else actual == expected - if not matches: - failures.append( - f"{example_name}: expected {key}={expected!r}, got {actual!r}" - ) - - -def expect_positive(example_name, value, key, failures): - actual = value.get(key) - try: - matches = actual > 0 - except TypeError: - matches = False - if not matches: - failures.append(f"{example_name}: expected {key} to be positive, got {actual!r}") - - -def main(): - try: - environment = example_environment() - except RuntimeError as error: - print(error, file=sys.stderr) - return 1 - - failures = [] - prepared = run_example("prepare", environment, failures) - result = run_example("run", environment, failures) - observed = run_example("observe", environment, failures) - - if prepared is not None: - expect_equal("prepare", prepared, "prompt_id", "support/prepare", failures) - expect_equal( - "prepare", prepared, "selected_profile", "support/fast", failures - ) - expect_equal("prepare", prepared, "model", "offline-model", failures) - expect_positive("prepare", prepared, "message_count", failures) - expect_equal("prepare", prepared, "profile_timeout_seconds", 30, failures) - expect_equal( - "prepare", prepared, "namespaces", ["review", "support"], failures - ) - expect_equal( - "prepare", - prepared, - "collections", - [ - {"namespace": "review", "release_version": None}, - {"namespace": "support", "release_version": "1.4.0"}, - ], - failures, - ) - expect_equal( - "prepare", prepared, "scoped_prompts", ["support/prepare"], failures - ) - expect_equal( - "prepare", prepared, "review_prompts", ["review/check"], failures - ) - - if result is not None: - expect_equal( - "run", result, "output", "Ada finished the migration review.", failures - ) - expect_equal("run", result, "is_valid", True, failures) - expect_equal("run", result, "validation_status", "passed", failures) - expect_equal("run", result, "model", "offline-model", failures) - expect_equal("run", result, "total_tokens", 18, failures) - - if observed is not None: - expect_equal("observe", observed, "proposal_attempts", 2, failures) - expect_equal( - "observe", observed, "proposal_transient_retries", 1, failures - ) - expect_equal( - "observe", observed, "proposal_usage_complete", False, failures - ) - expect_equal("observe", observed, "assessment_attempts", 1, failures) - expect_equal( - "observe", observed, "assessment_transient_retries", 0, failures - ) - expect_equal( - "observe", observed, "assessment_outcome", "accepted", failures - ) - - if failures: - print("Consumer example checks failed:", file=sys.stderr) - for failure in failures: - print(f"- {failure}", file=sys.stderr) - return 1 - - print("Consumer example checks passed: prepare, run, observe") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/test-release.py b/scripts/test-release.py index 428e4a6..1e6defa 100644 --- a/scripts/test-release.py +++ b/scripts/test-release.py @@ -1,7 +1,7 @@ """Check Promptkit's launcher and release profile using a disposable local remote. ReleaseKit owns the detailed publication, validation, and delivery algorithms. -These fixtures protect this repository's tool selection and product-check wiring. +These fixtures protect this repository's launcher and release-profile wiring. """ import json @@ -56,18 +56,7 @@ def prepare_fixture(root, environment, module_bytes): shutil.copy2(REPOSITORY_ROOT / "scripts" / name, destination) shutil.copy2(REPOSITORY_ROOT / "releasekit.json", repository / "releasekit.json") shutil.copy2(REPOSITORY_ROOT / ".gitignore", repository / ".gitignore") - for version in (RELEASE_VERSION, FAILED_VERSION): - write(repository / f"docs/releases/{version}.md", f"# Promptkit {version}\n\nFixture.\n") - write(repository / "scripts/check-examples.py", """ - import os - import sys - - with open(os.environ["FIXTURE_CHECK_MARKER"], "a") as marker: - marker.write("called\\n") - if os.environ.get("FIXTURE_CHECK_FAILURE"): - print("fixture example check rejected candidate", file=sys.stderr) - sys.exit(1) - """) + write(repository / f"docs/releases/{RELEASE_VERSION}.md", f"# Promptkit {RELEASE_VERSION}\n\nFixture.\n") write(repository / "fixture_test.go", """ package fixture @@ -127,14 +116,10 @@ def verify_publication(repository, remote, root, environment, profile, selected_ if git(remote, environment, "rev-parse", f"refs/tags/{RELEASE_VERSION}^{{}}") != candidate: raise RuntimeError("launcher did not publish the intended candidate") suite = Path(environment["FIXTURE_SUITE_MARKER"]).read_text().splitlines() - checks = Path(environment["FIXTURE_CHECK_MARKER"]).read_text().splitlines() - if suite != [selected_go] or checks != ["called"]: - raise RuntimeError(f"profile did not run the selected suite/check: {suite}, {checks}") + if suite != [selected_go]: + raise RuntimeError(f"profile did not run the selected suite: {suite}") - failed_environment = dict(environment, FIXTURE_CHECK_FAILURE="1") - run([publication, FAILED_VERSION], root, failed_environment, status=1) - if Path(environment["FIXTURE_CHECK_MARKER"]).read_text().splitlines() != ["called", "called"]: - raise RuntimeError("failure scenario did not exercise the product check") + run([publication, FAILED_VERSION], root, environment, status=1) run( ["git", "-C", str(remote), "show-ref", "--verify", "--quiet", f"refs/tags/{FAILED_VERSION}"], root, environment, status=1, @@ -153,14 +138,12 @@ def main(): GOPROXY="off", GONOPROXY="none", GOSUMDB="off") profile = json.loads((REPOSITORY_ROOT / "releasekit.json").read_text()) environment.pop(profile["delivery"].get("token_env", "RELEASEKIT_GITEA_TOKEN"), None) - environment.pop("FIXTURE_CHECK_FAILURE", None) module_bytes = {name: (REPOSITORY_ROOT / name).read_bytes() for name in ("go.mod", "go.sum")} try: selected_go = run(["go", "env", "GOVERSION"], REPOSITORY_ROOT, environment).strip() with tempfile.TemporaryDirectory(prefix="promptkit-release-fixture-") as directory: root = Path(directory) - environment.update(FIXTURE_CHECK_MARKER=str(root / "checks"), - FIXTURE_SUITE_MARKER=str(root / "suite")) + environment.update(FIXTURE_SUITE_MARKER=str(root / "suite")) repository, remote = prepare_fixture(root, environment, module_bytes) verify_bootstrap(repository, root, environment, selected_go) verify_publication(repository, remote, root, environment, profile, selected_go)