base-layer/promptkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Remove the custom example release check

Browse Source
This commit is contained in:
Eric Rakestraw
2026-11-03 01:38:30 +00:00
parent 66b98831d3
commit 175140ae5a
4 changed files with 13 additions and 218 deletions
Download Patch File Download Diff File Expand all files Collapse all files

181
scripts/check-examples.py
View File

@@ -1,181 +0,0 @@
#!/usr/bin/env python3
"""Run the maintained consumer examples and check their semantic output."""
import json
import os
import subprocess
import sys
from pathlib import Path
REPOSITORY_ROOT = Path(__file__).resolve().parent.parent
def example_environment():
environment = os.environ.copy()
environment["GOTOOLCHAIN"] = "local"
try:
host = subprocess.run(
["go", "env", "GOHOSTOS", "GOHOSTARCH"],
cwd=REPOSITORY_ROOT,
env=environment,
text=True,
capture_output=True,
check=False,
)
except OSError as error:
raise RuntimeError(f"could not run go env: {error}") from error
if host.returncode != 0:
raise RuntimeError(
"go env GOHOSTOS GOHOSTARCH failed "
f"with exit status {host.returncode}: {host.stderr.strip()}"
)
values = host.stdout.splitlines()
if len(values) != 2 or not all(values):
raise RuntimeError(
"go env GOHOSTOS GOHOSTARCH returned unexpected output: "
f"{host.stdout!r}"
)
environment["GOOS"], environment["GOARCH"] = values
return environment
def run_example(name, environment, failures):
command = ["go", "run", f"./examples/go-library/{name}"]
try:
result = subprocess.run(
command,
cwd=REPOSITORY_ROOT,
env=environment,
text=True,
capture_output=True,
check=False,
)
except OSError as error:
failures.append(f"{name}: could not start {command[0]}: {error}")
return None
if result.returncode != 0:
failures.append(f"{name}: command exited with status {result.returncode}")
if result.stderr:
failures.append(f"{name}: stderr:\n{result.stderr.rstrip()}")
if result.stdout:
failures.append(f"{name}: stdout:\n{result.stdout.rstrip()}")
return None
try:
value = json.loads(result.stdout)
except json.JSONDecodeError as error:
failures.append(f"{name}: output is not valid JSON: {error}")
if result.stdout:
failures.append(f"{name}: stdout:\n{result.stdout.rstrip()}")
return None
if not isinstance(value, dict):
failures.append(
f"{name}: expected a JSON object, got {type(value).__name__}"
)
return None
return value
def expect_equal(example_name, value, key, expected, failures):
actual = value.get(key)
matches = actual is expected if isinstance(expected, bool) else actual == expected
if not matches:
failures.append(
f"{example_name}: expected {key}={expected!r}, got {actual!r}"
)
def expect_positive(example_name, value, key, failures):
actual = value.get(key)
try:
matches = actual > 0
except TypeError:
matches = False
if not matches:
failures.append(f"{example_name}: expected {key} to be positive, got {actual!r}")
def main():
try:
environment = example_environment()
except RuntimeError as error:
print(error, file=sys.stderr)
return 1
failures = []
prepared = run_example("prepare", environment, failures)
result = run_example("run", environment, failures)
observed = run_example("observe", environment, failures)
if prepared is not None:
expect_equal("prepare", prepared, "prompt_id", "support/prepare", failures)
expect_equal(
"prepare", prepared, "selected_profile", "support/fast", failures
)
expect_equal("prepare", prepared, "model", "offline-model", failures)
expect_positive("prepare", prepared, "message_count", failures)
expect_equal("prepare", prepared, "profile_timeout_seconds", 30, failures)
expect_equal(
"prepare", prepared, "namespaces", ["review", "support"], failures
)
expect_equal(
"prepare",
prepared,
"collections",
[
{"namespace": "review", "release_version": None},
{"namespace": "support", "release_version": "1.4.0"},
],
failures,
)
expect_equal(
"prepare", prepared, "scoped_prompts", ["support/prepare"], failures
)
expect_equal(
"prepare", prepared, "review_prompts", ["review/check"], failures
)
if result is not None:
expect_equal(
"run", result, "output", "Ada finished the migration review.", failures
)
expect_equal("run", result, "is_valid", True, failures)
expect_equal("run", result, "validation_status", "passed", failures)
expect_equal("run", result, "model", "offline-model", failures)
expect_equal("run", result, "total_tokens", 18, failures)
if observed is not None:
expect_equal("observe", observed, "proposal_attempts", 2, failures)
expect_equal(
"observe", observed, "proposal_transient_retries", 1, failures
)
expect_equal(
"observe", observed, "proposal_usage_complete", False, failures
)
expect_equal("observe", observed, "assessment_attempts", 1, failures)
expect_equal(
"observe", observed, "assessment_transient_retries", 0, failures
)
expect_equal(
"observe", observed, "assessment_outcome", "accepted", failures
)
if failures:
print("Consumer example checks failed:", file=sys.stderr)
for failure in failures:
print(f"- {failure}", file=sys.stderr)
return 1
print("Consumer example checks passed: prepare, run, observe")
return 0
if __name__ == "__main__":
raise SystemExit(main())

29
scripts/test-release.py
View File

@@ -1,7 +1,7 @@
"""Check Promptkit's launcher and release profile using a disposable local remote.
ReleaseKit owns the detailed publication, validation, and delivery algorithms.
These fixtures protect this repository's tool selection and product-check wiring.
These fixtures protect this repository's launcher and release-profile wiring.
"""
import json
@@ -56,18 +56,7 @@ def prepare_fixture(root, environment, module_bytes):
shutil.copy2(REPOSITORY_ROOT / "scripts" / name, destination)
shutil.copy2(REPOSITORY_ROOT / "releasekit.json", repository / "releasekit.json")
shutil.copy2(REPOSITORY_ROOT / ".gitignore", repository / ".gitignore")
for version in (RELEASE_VERSION, FAILED_VERSION):
write(repository / f"docs/releases/{version}.md", f"# Promptkit {version}\n\nFixture.\n")
write(repository / "scripts/check-examples.py", """
import os
import sys
with open(os.environ["FIXTURE_CHECK_MARKER"], "a") as marker:
marker.write("called\\n")
if os.environ.get("FIXTURE_CHECK_FAILURE"):
print("fixture example check rejected candidate", file=sys.stderr)
sys.exit(1)
""")
write(repository / f"docs/releases/{RELEASE_VERSION}.md", f"# Promptkit {RELEASE_VERSION}\n\nFixture.\n")
write(repository / "fixture_test.go", """
package fixture
@@ -127,14 +116,10 @@ def verify_publication(repository, remote, root, environment, profile, selected_
if git(remote, environment, "rev-parse", f"refs/tags/{RELEASE_VERSION}^{{}}") != candidate:
raise RuntimeError("launcher did not publish the intended candidate")
suite = Path(environment["FIXTURE_SUITE_MARKER"]).read_text().splitlines()
checks = Path(environment["FIXTURE_CHECK_MARKER"]).read_text().splitlines()
if suite != [selected_go] or checks != ["called"]:
raise RuntimeError(f"profile did not run the selected suite/check: {suite}, {checks}")
if suite != [selected_go]:
raise RuntimeError(f"profile did not run the selected suite: {suite}")
failed_environment = dict(environment, FIXTURE_CHECK_FAILURE="1")
run([publication, FAILED_VERSION], root, failed_environment, status=1)
if Path(environment["FIXTURE_CHECK_MARKER"]).read_text().splitlines() != ["called", "called"]:
raise RuntimeError("failure scenario did not exercise the product check")
run([publication, FAILED_VERSION], root, environment, status=1)
run(
["git", "-C", str(remote), "show-ref", "--verify", "--quiet", f"refs/tags/{FAILED_VERSION}"],
root, environment, status=1,
@@ -153,14 +138,12 @@ def main():
GOPROXY="off", GONOPROXY="none", GOSUMDB="off")
profile = json.loads((REPOSITORY_ROOT / "releasekit.json").read_text())
environment.pop(profile["delivery"].get("token_env", "RELEASEKIT_GITEA_TOKEN"), None)
environment.pop("FIXTURE_CHECK_FAILURE", None)
module_bytes = {name: (REPOSITORY_ROOT / name).read_bytes() for name in ("go.mod", "go.sum")}
try:
selected_go = run(["go", "env", "GOVERSION"], REPOSITORY_ROOT, environment).strip()
with tempfile.TemporaryDirectory(prefix="promptkit-release-fixture-") as directory:
root = Path(directory)
environment.update(FIXTURE_CHECK_MARKER=str(root / "checks"),
FIXTURE_SUITE_MARKER=str(root / "suite"))
environment.update(FIXTURE_SUITE_MARKER=str(root / "suite"))
repository, remote = prepare_fixture(root, environment, module_bytes)
verify_bootstrap(repository, root, environment, selected_go)
verify_publication(repository, remote, root, environment, profile, selected_go)