From e256e6455155ccceed9a5e70f8397f0730d1ecc5 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Mon, 9 Sep 2024 22:27:54 -0700 Subject: [PATCH] Package Pex `--scies` for release. These are likely better than the Pex PEX for most Pex binary use cases since they include the management extra and run in `--venv` mode as well as providing BusyBox support for equal footing of the `pex`, `pex-tools` and `pex3` top-level commands. --- .github/workflows/release.yml | 10 +- .../pex_build/hatchling/build_hook.py | 2 +- package/__init__.py | 2 + package/scie_config.py | 25 ++++ ...ld_cache_image.py => build-cache-image.py} | 0 scripts/{build_docs.py => build-docs.py} | 0 scripts/{package.py => create-packages.py} | 125 ++++++++++++++++++ ...mbed_virtualenv.py => embed-virtualenv.py} | 0 ..._scie_platform.py => gen-scie-platform.py} | 23 +--- tests/integration/test_issue_1872.py | 24 +++- tests/integration/test_issue_1995.py | 10 +- tox.ini | 13 +- 12 files changed, 193 insertions(+), 41 deletions(-) create mode 100644 package/__init__.py create mode 100644 package/scie_config.py rename scripts/{build_cache_image.py => build-cache-image.py} (100%) rename scripts/{build_docs.py => build-docs.py} (100%) rename scripts/{package.py => create-packages.py} (61%) rename scripts/{embed_virtualenv.py => embed-virtualenv.py} (100%) rename scripts/{gen_scie_platform.py => gen-scie-platform.py} (94%) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c99e12646..116153bea 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -88,7 +88,7 @@ jobs: - name: Package Pex ${{ needs.determine-tag.outputs.release-tag }} PEX uses: pantsbuild/actions/run-tox@b16b9cf47cd566acfe217b1dafc5b452e27e6fd7 with: - tox-env: package -- --embed-docs + tox-env: package -- --embed-docs --scies --gen-md-table-of-hash-and-size dist/hashes.md - name: Generate Pex ${{ needs.determine-tag.outputs.release-tag }} PDF uses: pantsbuild/actions/run-tox@b16b9cf47cd566acfe217b1dafc5b452e27e6fd7 with: @@ -97,7 +97,7 @@ jobs: uses: actions/attest-build-provenance@v1 with: subject-path: | - dist/pex + dist/pex* dist/docs/pdf/pex.pdf - name: Prepare Changelog id: prepare-changelog @@ -105,6 +105,12 @@ jobs: with: changelog-file: ${{ github.workspace }}/CHANGES.md version: ${{ needs.determine-tag.outputs.release-version }} + - name: Append Hashes to Changelog + run: | + changelog_tmp="$(mktemp)" + cat "${{ steps.prepare-changelog.outputs.changelog-file }}" <(echo '***') dist/hashes.md \ + > "${changelog_tmp}" + mv "${changelog_tmp}" "${{ steps.prepare-changelog.outputs.changelog-file }}" - name: Create ${{ needs.determine-tag.outputs.release-tag }} Release uses: softprops/action-gh-release@v2 with: diff --git a/build-backend/pex_build/hatchling/build_hook.py b/build-backend/pex_build/hatchling/build_hook.py index 1b89faeda..f0beb03c2 100644 --- a/build-backend/pex_build/hatchling/build_hook.py +++ b/build-backend/pex_build/hatchling/build_hook.py @@ -30,7 +30,7 @@ def initialize( subprocess.check_call( args=[ sys.executable, - os.path.join(self.root, "scripts", "build_docs.py"), + os.path.join(self.root, "scripts", "build-docs.py"), "--clean-html", out_dir, ] diff --git a/package/__init__.py b/package/__init__.py new file mode 100644 index 000000000..87fb2ed9a --- /dev/null +++ b/package/__init__.py @@ -0,0 +1,2 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). diff --git a/package/scie_config.py b/package/scie_config.py new file mode 100644 index 000000000..3489570a5 --- /dev/null +++ b/package/scie_config.py @@ -0,0 +1,25 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +import pkgutil +from dataclasses import dataclass + +import toml + + +@dataclass(frozen=True) +class ScieConfig: + @classmethod + def load(cls, *, pbs_release: str | None = None, python_version: str | None = None): + scie_config = toml.loads(pkgutil.get_data(__name__, "package.toml").decode())["scie"] + return cls( + pbs_release=pbs_release or scie_config["pbs-release"], + python_version=python_version or scie_config["python-version"], + pex_extras=tuple(scie_config["pex-extras"]), + platforms=tuple(scie_config["platforms"]), + ) + + pbs_release: str + python_version: str + pex_extras: tuple[str, ...] + platforms: tuple[str, ...] diff --git a/scripts/build_cache_image.py b/scripts/build-cache-image.py similarity index 100% rename from scripts/build_cache_image.py rename to scripts/build-cache-image.py diff --git a/scripts/build_docs.py b/scripts/build-docs.py similarity index 100% rename from scripts/build_docs.py rename to scripts/build-docs.py diff --git a/scripts/package.py b/scripts/create-packages.py similarity index 61% rename from scripts/package.py rename to scripts/create-packages.py index d7e6a6e58..f5cd21403 100755 --- a/scripts/package.py +++ b/scripts/create-packages.py @@ -1,6 +1,9 @@ #!/usr/bin/env python3 +from __future__ import absolute_import, annotations + import atexit +import glob import hashlib import io import os @@ -15,7 +18,11 @@ from pathlib import Path, PurePath from typing import Dict, Iterator, Optional, Tuple, cast +from package.scie_config import ScieConfig +from pex.common import safe_mkdtemp + DIST_DIR = Path("dist") +PACKAGE_DIR = Path("package") def build_pex_pex( @@ -53,6 +60,85 @@ def build_pex_pex( return output_file +def build_pex_scies( + scie_dest_dir: Path, verbosity: int = 0, env: Optional[Dict[str, str]] = None +) -> Iterator[tuple[Path, str]]: + scie_config = ScieConfig.load() + + pex_requirement = f".[{','.join(scie_config.pex_extras)}]" + + lock = PACKAGE_DIR / "pex-scie.lock" + if not lock.exists(): + raise SystemExit("XXX") + + missing_platforms: list[str] = [] + platforms: list[tuple[str, Path]] = [] + for platform in scie_config.platforms: + complete_platform = PACKAGE_DIR / "complete-platforms" / f"{platform}.json" + if not complete_platform.exists(): + missing_platforms.append(platform) + else: + platforms.append((platform, complete_platform)) + if missing_platforms: + raise SystemExit("YYY") + + for platform, complete_platform in platforms: + dest_dir = safe_mkdtemp() + output_file = os.path.join(dest_dir, "pex") + args = [ + sys.executable, + "-m", + "pex", + *["-v" for _ in range(verbosity)], + "--disable-cache", + "--no-build", + "--no-compile", + "--no-use-system-time", + "--venv", + "--no-strip-pex-env", + "--complete-platform", + str(complete_platform), + "--lock", + str(lock), + "--scie", + "eager", + "--scie-only", + "--scie-name-style", + "platform-file-suffix", + "--scie-platform", + platform, + "--scie-pbs-release", + scie_config.pbs_release, + "--scie-python-version", + scie_config.python_version, + "--scie-pbs-stripped", + "--scie-hash-alg", + "sha256", + "--scie-busybox", + "@pex", + "-o", + output_file, + "-c", + "pex", + "--project", + pex_requirement, + ] + subprocess.run(args=args, env=env, check=True) + + artifacts = glob.glob(f"{output_file}*") + scie_artifacts = [artifact for artifact in artifacts if not artifact.endswith(".sha256")] + if len(scie_artifacts) != 1: + raise SystemExit( + f"Found unexpected artifacts after generating Pex scie:{os.linesep}" + f"{os.linesep.join(sorted(artifacts))}" + ) + scie_name = os.path.basename(scie_artifacts[0]) + for artifact in artifacts: + shutil.move(artifact, scie_dest_dir / os.path.basename(artifact)) + + yield scie_dest_dir / scie_name, platform + + def describe_rev() -> str: if not os.path.isdir(".git") and os.path.isfile("PKG-INFO"): # We're being build from an unpacked sdist. @@ -128,22 +214,41 @@ def main( embed_docs: bool = False, clean_docs: bool = False, pex_output_file: Optional[Path] = DIST_DIR / "pex", + scie_dest_dir: Optional[Path] = None, + markdown_hash_table_file: Optional[Path] = None, serve: bool = False ) -> None: env = os.environ.copy() if embed_docs: env.update(__PEX_BUILD_INCLUDE_DOCS__="1") + hash_table: dict[Path, tuple[str, int]] = {} if pex_output_file: print(f"Building Pex PEX to `{pex_output_file}` ...") build_pex_pex(pex_output_file, verbosity, env=env) rev = describe_rev() sha256, size = describe_file(pex_output_file) + hash_table[pex_output_file] = sha256, size print(f"Built Pex PEX @ {rev}:") print(f"sha256: {sha256}") print(f" size: {size}") + if scie_dest_dir: + print(f"Building Pex scies to `{scie_dest_dir}` ...") + for scie, platform in build_pex_scies(scie_dest_dir, verbosity, env=env): + hash_table[scie] = describe_file(scie) + print(f" Built Pex scie for {platform} at `{scie}`") + + if markdown_hash_table_file and hash_table: + with markdown_hash_table_file.open(mode="w") as fp: + print("|file|sha256|size|", file=fp) + print("|----|------|----|", file=fp) + for file, (sha256, size) in sorted(hash_table.items()): + print(f"|{file.name}|{sha256}|{size}|", file=fp) + + print(f"Generated markdown table of Pex sizes & hashes at `{markdown_hash_table_file}`") + if additional_dist_formats: print( f"Building additional distribution formats to `{DIST_DIR}`: " @@ -214,6 +319,24 @@ def main( type=Path, help="Build the Pex PEX at this path.", ) + parser.add_argument( + "--scies", + default=False, + action="store_true", + help="Build PEX scies.", + ) + parser.add_argument( + "--scie-dest-dir", + default=DIST_DIR, + type=Path, + help="Build the Pex scies in this dir.", + ) + parser.add_argument( + "--gen-md-table-of-hash-and-size", + default=None, + type=Path, + help="A path to generate a markdown table of packaged asset hashes to.", + ) parser.add_argument( "--serve", default=False, @@ -228,5 +351,7 @@ def main( embed_docs=args.embed_docs, clean_docs=args.clean_docs, pex_output_file=None if args.no_pex else args.pex_output_file, + scie_dest_dir=args.scie_dest_dir if args.scies else None, + markdown_hash_table_file=args.gen_md_table_of_hash_and_size, serve=args.serve ) diff --git a/scripts/embed_virtualenv.py b/scripts/embed-virtualenv.py similarity index 100% rename from scripts/embed_virtualenv.py rename to scripts/embed-virtualenv.py diff --git a/scripts/gen_scie_platform.py b/scripts/gen-scie-platform.py similarity index 94% rename from scripts/gen_scie_platform.py rename to scripts/gen-scie-platform.py index e39471963..70b00a771 100644 --- a/scripts/gen_scie_platform.py +++ b/scripts/gen-scie-platform.py @@ -15,17 +15,17 @@ import zipfile from argparse import ArgumentError, ArgumentTypeError from contextlib import contextmanager -from dataclasses import dataclass from pathlib import Path from textwrap import dedent from typing import IO, Collection, Iterable, Iterator import github import httpx -import toml from github import Github from github.WorkflowRun import WorkflowRun +from package.scie_config import ScieConfig + logger = logging.getLogger(__name__) @@ -37,25 +37,6 @@ class GitHubError(Exception): GEN_SCIE_PLATFORMS_WORKFLOW = "gen-scie-platforms.yml" -@dataclass(frozen=True) -class ScieConfig: - @classmethod - def load(cls, *, pbs_release: str | None = None, python_version: str | None = None): - with (PACKAGE_DIR / "package.toml").open() as fp: - scie_config = toml.load(fp)["scie"] - return cls( - pbs_release=pbs_release or scie_config["pbs-release"], - python_version=python_version or scie_config["python-version"], - pex_extras=tuple(scie_config["pex-extras"]), - platforms=tuple(scie_config["platforms"]), - ) - - pbs_release: str - python_version: str - pex_extras: tuple[str, ...] - platforms: tuple[str, ...] - - def create_all_complete_platforms( dest_dir: Path, scie_config: ScieConfig, diff --git a/tests/integration/test_issue_1872.py b/tests/integration/test_issue_1872.py index 3bb85dcba..3b6aa9a78 100644 --- a/tests/integration/test_issue_1872.py +++ b/tests/integration/test_issue_1872.py @@ -5,7 +5,6 @@ import subprocess import sys -from pex.compatibility import PY3 from pex.pep_440 import Version from pex.pep_503 import ProjectName from pex.resolve.locked_resolve import LocalProjectArtifact @@ -13,7 +12,7 @@ from pex.resolve.resolved_requirement import Pin from pex.typing import TYPE_CHECKING from pex.version import __version__ -from testing import PY38, ensure_python_interpreter, make_env +from testing import PY310, ensure_python_interpreter, make_env, run_pex_command if TYPE_CHECKING: from typing import Any @@ -25,12 +24,23 @@ def test_pep_518_venv_pex_env_scrubbing( ): # type: (...) -> None - # N.B.: The package script requires Python 3. - python = sys.executable if PY3 else ensure_python_interpreter(PY38) - - package_script = os.path.join(pex_project_dir, "scripts", "package.py") pex_pex = os.path.join(str(tmpdir), "pex") - subprocess.check_call(args=[python, package_script, "--pex-output-file", pex_pex]) + package_script = os.path.join(pex_project_dir, "scripts", "create-packages.py") + run_pex_command( + args=[ + "toml", + pex_project_dir, + "--", + package_script, + "-v", + "--pex-output-file", + pex_pex, + ], + # The package script requires Python>=3.8. + python=( + sys.executable if sys.version_info[:2] >= (3, 8) else ensure_python_interpreter(PY310) + ), + ).assert_success() lock = os.path.join(str(tmpdir), "lock.json") subprocess.check_call( diff --git a/tests/integration/test_issue_1995.py b/tests/integration/test_issue_1995.py index 6af3aaf69..421c0e138 100644 --- a/tests/integration/test_issue_1995.py +++ b/tests/integration/test_issue_1995.py @@ -8,7 +8,7 @@ import pytest from pex.typing import TYPE_CHECKING -from testing import IS_LINUX, PY38, ensure_python_interpreter, make_env, run_pex_command +from testing import IS_LINUX, PY310, ensure_python_interpreter, make_env, run_pex_command if TYPE_CHECKING: from typing import Any @@ -21,7 +21,7 @@ def test_packaging( ): # type: (...) -> None pex = os.path.join(str(tmpdir), "pex.pex") - package_script = os.path.join(pex_project_dir, "scripts", "package.py") + package_script = os.path.join(pex_project_dir, "scripts", "create-packages.py") run_pex_command( args=[ "toml", @@ -32,8 +32,10 @@ def test_packaging( "--pex-output-file", pex, ], - # The package script requires Python 3. - python=sys.executable if sys.version_info[0] >= 3 else ensure_python_interpreter(PY38), + # The package script requires Python>=3.8. + python=( + sys.executable if sys.version_info[:2] >= (3, 8) else ensure_python_interpreter(PY310) + ), ).assert_success() assert os.path.exists(pex), "Expected {pex} to be created by {package_script}.".format( pex=pex, package_script=package_script diff --git a/tox.ini b/tox.ini index ccae821b0..41572a7ec 100644 --- a/tox.ini +++ b/tox.ini @@ -199,7 +199,7 @@ deps = httpx==0.23.0 commands = tox -e vendor -- --no-update - python scripts/embed_virtualenv.py + python scripts/embed-virtualenv.py git diff --exit-code [testenv:docs] @@ -207,7 +207,7 @@ basepython = python3 deps = -r docs-requirements.txt commands = - python scripts/build_docs.py {posargs} + python scripts/build-docs.py {posargs} [testenv:gen-scie-platform] basepython = python3 @@ -217,7 +217,7 @@ deps = toml==0.10.2 PyGithub==2.4.0 commands = - python scripts/gen_scie_platform.py {posargs} + python scripts/gen-scie-platform.py {posargs} [_package] basepython = python3 @@ -227,6 +227,7 @@ deps = # through either improved or broken isolation + backend-path integration; so we pin low. # See: https://github.com/pypa/pyproject-hooks/pull/165 pyproject-hooks<1.1.0 + toml==0.10.2 [testenv:package] skip_install = true @@ -234,7 +235,7 @@ basepython = {[_package]basepython} deps = {[_package]deps} commands = - python scripts/package.py {posargs} + python scripts/create-packages.py {posargs} [testenv:serve] skip_install = true @@ -242,7 +243,7 @@ basepython = {[_package]basepython} deps = {[_package]deps} commands = - python scripts/package.py --additional-format wheel --local --serve {posargs} + python scripts/create-packages.py --additional-format wheel --local --serve {posargs} [testenv:pip] description = Run Pex's vendored pip. @@ -284,4 +285,4 @@ deps = coloredlogs==15.0.1 PyYAML==6.0.1 commands = - python scripts/build_cache_image.py {posargs} + python scripts/build-cache-image.py {posargs}