Package Pex --scies for release.

These are likely better than the Pex PEX for most Pex binary use cases
since they include the management extra and run in `--venv` mode as well
as providing BusyBox support for equal footing of the `pex`, `pex-tools`
and `pex3` top-level commands.

.github/workflows/release.yml

@@ -88,7 +88,7 @@ jobs:
       - name: Package Pex ${{ needs.determine-tag.outputs.release-tag }} PEX
         uses: pantsbuild/actions/run-tox@b16b9cf47cd566acfe217b1dafc5b452e27e6fd7
         with:
-          tox-env: package -- --embed-docs
+          tox-env: package -- --embed-docs --scies --gen-md-table-of-hash-and-size dist/hashes.md
       - name: Generate Pex ${{ needs.determine-tag.outputs.release-tag }} PDF
         uses: pantsbuild/actions/run-tox@b16b9cf47cd566acfe217b1dafc5b452e27e6fd7
         with:
@@ -97,14 +97,20 @@ jobs:
         uses: actions/attest-build-provenance@v1
         with:
           subject-path: |
-            dist/pex
+            dist/pex*
             dist/docs/pdf/pex.pdf
       - name: Prepare Changelog
         id: prepare-changelog
         uses: a-scie/actions/changelog@v1.6
         with:
           changelog-file: ${{ github.workspace }}/CHANGES.md
           version: ${{ needs.determine-tag.outputs.release-version }}
+      - name: Append Hashes to Changelog
+        run: |
+          changelog_tmp="$(mktemp)"
+          cat "${{ steps.prepare-changelog.outputs.changelog-file }}" <(echo '***') dist/hashes.md \
+            > "${changelog_tmp}"
+          mv "${changelog_tmp}" "${{ steps.prepare-changelog.outputs.changelog-file }}"
       - name: Create ${{ needs.determine-tag.outputs.release-tag }} Release
         uses: softprops/action-gh-release@v2
         with:

build-backend/pex_build/hatchling/build_hook.py

@@ -30,7 +30,7 @@ def initialize(
             subprocess.check_call(
                 args=[
                     sys.executable,
-                    os.path.join(self.root, "scripts", "build_docs.py"),
+                    os.path.join(self.root, "scripts", "build-docs.py"),
                     "--clean-html",
                     out_dir,
                 ]

package/__init__.py

@@ -0,0 +1,2 @@
+# Copyright 2024 Pex project contributors.
+# Licensed under the Apache License, Version 2.0 (see LICENSE).

package/scie_config.py

@@ -0,0 +1,25 @@
+# Copyright 2024 Pex project contributors.
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+import pkgutil
+from dataclasses import dataclass
+
+import toml
+
+
+@dataclass(frozen=True)
+class ScieConfig:
+    @classmethod
+    def load(cls, *, pbs_release: str | None = None, python_version: str | None = None):
+        scie_config = toml.loads(pkgutil.get_data(__name__, "package.toml").decode())["scie"]
+        return cls(
+            pbs_release=pbs_release or scie_config["pbs-release"],
+            python_version=python_version or scie_config["python-version"],
+            pex_extras=tuple(scie_config["pex-extras"]),
+            platforms=tuple(scie_config["platforms"]),
+        )
+
+    pbs_release: str
+    python_version: str
+    pex_extras: tuple[str, ...]
+    platforms: tuple[str, ...]

scripts/package.py → scripts/create-packages.py

@@ -1,6 +1,9 @@
 #!/usr/bin/env python3
 
+from __future__ import absolute_import, annotations
+
 import atexit
+import glob
 import hashlib
 import io
 import os
@@ -15,7 +18,11 @@
 from pathlib import Path, PurePath
 from typing import Dict, Iterator, Optional, Tuple, cast
 
+from package.scie_config import ScieConfig
+from pex.common import safe_mkdtemp
+
 DIST_DIR = Path("dist")
+PACKAGE_DIR = Path("package")
 
 
 def build_pex_pex(
@@ -53,6 +60,85 @@ def build_pex_pex(
     return output_file
 
 
+def build_pex_scies(
+    scie_dest_dir: Path, verbosity: int = 0, env: Optional[Dict[str, str]] = None
+) -> Iterator[tuple[Path, str]]:
+    scie_config = ScieConfig.load()
+
+    pex_requirement = f".[{','.join(scie_config.pex_extras)}]"
+
+    lock = PACKAGE_DIR / "pex-scie.lock"
+    if not lock.exists():
+        raise SystemExit("XXX")
+
+    missing_platforms: list[str] = []
+    platforms: list[tuple[str, Path]] = []
+    for platform in scie_config.platforms:
+        complete_platform = PACKAGE_DIR / "complete-platforms" / f"{platform}.json"
+        if not complete_platform.exists():
+            missing_platforms.append(platform)
+        else:
+            platforms.append((platform, complete_platform))
+    if missing_platforms:
+        raise SystemExit("YYY")
+
+    for platform, complete_platform in platforms:
+        dest_dir = safe_mkdtemp()
+        output_file = os.path.join(dest_dir, "pex")
+        args = [
+            sys.executable,
+            "-m",
+            "pex",
+            *["-v" for _ in range(verbosity)],
+            "--disable-cache",
+            "--no-build",
+            "--no-compile",
+            "--no-use-system-time",
+            "--venv",
+            "--no-strip-pex-env",
+            "--complete-platform",
+            str(complete_platform),
+            "--lock",
+            str(lock),
+            "--scie",
+            "eager",
+            "--scie-only",
+            "--scie-name-style",
+            "platform-file-suffix",
+            "--scie-platform",
+            platform,
+            "--scie-pbs-release",
+            scie_config.pbs_release,
+            "--scie-python-version",
+            scie_config.python_version,
+            "--scie-pbs-stripped",
+            "--scie-hash-alg",
+            "sha256",
+            "--scie-busybox",
+            "@pex",
+            "-o",
+            output_file,
+            "-c",
+            "pex",
+            "--project",
+            pex_requirement,
+        ]
+        subprocess.run(args=args, env=env, check=True)
+
+        artifacts = glob.glob(f"{output_file}*")
+        scie_artifacts = [artifact for artifact in artifacts if not artifact.endswith(".sha256")]
+        if len(scie_artifacts) != 1:
+            raise SystemExit(
+                f"Found unexpected artifacts after generating Pex scie:{os.linesep}"
+                f"{os.linesep.join(sorted(artifacts))}"
+            )
+        scie_name = os.path.basename(scie_artifacts[0])
+        for artifact in artifacts:
+            shutil.move(artifact, scie_dest_dir / os.path.basename(artifact))
+
+        yield scie_dest_dir / scie_name, platform
+
+
 def describe_rev() -> str:
     if not os.path.isdir(".git") and os.path.isfile("PKG-INFO"):
         # We're being build from an unpacked sdist.
@@ -128,22 +214,41 @@ def main(
     embed_docs: bool = False,
     clean_docs: bool = False,
     pex_output_file: Optional[Path] = DIST_DIR / "pex",
+    scie_dest_dir: Optional[Path] = None,
+    markdown_hash_table_file: Optional[Path] = None,
     serve: bool = False
 ) -> None:
     env = os.environ.copy()
     if embed_docs:
         env.update(__PEX_BUILD_INCLUDE_DOCS__="1")
 
+    hash_table: dict[Path, tuple[str, int]] = {}
     if pex_output_file:
         print(f"Building Pex PEX to `{pex_output_file}` ...")
         build_pex_pex(pex_output_file, verbosity, env=env)
 
         rev = describe_rev()
         sha256, size = describe_file(pex_output_file)
+        hash_table[pex_output_file] = sha256, size
         print(f"Built Pex PEX @ {rev}:")
         print(f"sha256: {sha256}")
         print(f"  size: {size}")
 
+    if scie_dest_dir:
+        print(f"Building Pex scies to `{scie_dest_dir}` ...")
+        for scie, platform in build_pex_scies(scie_dest_dir, verbosity, env=env):
+            hash_table[scie] = describe_file(scie)
+            print(f"  Built Pex scie for {platform} at `{scie}`")
+
+    if markdown_hash_table_file and hash_table:
+        with markdown_hash_table_file.open(mode="w") as fp:
+            print("|file|sha256|size|", file=fp)
+            print("|----|------|----|", file=fp)
+            for file, (sha256, size) in sorted(hash_table.items()):
+                print(f"|{file.name}|{sha256}|{size}|", file=fp)
+
+        print(f"Generated markdown table of Pex sizes & hashes at `{markdown_hash_table_file}`")
+
     if additional_dist_formats:
         print(
             f"Building additional distribution formats to `{DIST_DIR}`: "
@@ -214,6 +319,24 @@ def main(
         type=Path,
         help="Build the Pex PEX at this path.",
     )
+    parser.add_argument(
+        "--scies",
+        default=False,
+        action="store_true",
+        help="Build PEX scies.",
+    )
+    parser.add_argument(
+        "--scie-dest-dir",
+        default=DIST_DIR,
+        type=Path,
+        help="Build the Pex scies in this dir.",
+    )
+    parser.add_argument(
+        "--gen-md-table-of-hash-and-size",
+        default=None,
+        type=Path,
+        help="A path to generate a markdown table of packaged asset hashes to.",
+    )
     parser.add_argument(
         "--serve",
         default=False,
@@ -228,5 +351,7 @@ def main(
         embed_docs=args.embed_docs,
         clean_docs=args.clean_docs,
         pex_output_file=None if args.no_pex else args.pex_output_file,
+        scie_dest_dir=args.scie_dest_dir if args.scies else None,
+        markdown_hash_table_file=args.gen_md_table_of_hash_and_size,
         serve=args.serve
     )

scripts/gen_scie_platform.py → scripts/gen-scie-platform.py

@@ -15,17 +15,17 @@
 import zipfile
 from argparse import ArgumentError, ArgumentTypeError
 from contextlib import contextmanager
-from dataclasses import dataclass
 from pathlib import Path
 from textwrap import dedent
 from typing import IO, Collection, Iterable, Iterator
 
 import github
 import httpx
-import toml
 from github import Github
 from github.WorkflowRun import WorkflowRun
 
+from package.scie_config import ScieConfig
+
 logger = logging.getLogger(__name__)
 
 
@@ -37,25 +37,6 @@ class GitHubError(Exception):
 GEN_SCIE_PLATFORMS_WORKFLOW = "gen-scie-platforms.yml"
 
 
-@dataclass(frozen=True)
-class ScieConfig:
-    @classmethod
-    def load(cls, *, pbs_release: str | None = None, python_version: str | None = None):
-        with (PACKAGE_DIR / "package.toml").open() as fp:
-            scie_config = toml.load(fp)["scie"]
-        return cls(
-            pbs_release=pbs_release or scie_config["pbs-release"],
-            python_version=python_version or scie_config["python-version"],
-            pex_extras=tuple(scie_config["pex-extras"]),
-            platforms=tuple(scie_config["platforms"]),
-        )
-
-    pbs_release: str
-    python_version: str
-    pex_extras: tuple[str, ...]
-    platforms: tuple[str, ...]
-
-
 def create_all_complete_platforms(
     dest_dir: Path,
     scie_config: ScieConfig,

tests/integration/test_issue_1872.py

@@ -5,15 +5,14 @@
 import subprocess
 import sys
 
-from pex.compatibility import PY3
 from pex.pep_440 import Version
 from pex.pep_503 import ProjectName
 from pex.resolve.locked_resolve import LocalProjectArtifact
 from pex.resolve.lockfile import json_codec
 from pex.resolve.resolved_requirement import Pin
 from pex.typing import TYPE_CHECKING
 from pex.version import __version__
-from testing import PY38, ensure_python_interpreter, make_env
+from testing import PY310, ensure_python_interpreter, make_env, run_pex_command
 
 if TYPE_CHECKING:
     from typing import Any
@@ -25,12 +24,23 @@ def test_pep_518_venv_pex_env_scrubbing(
 ):
     # type: (...) -> None
 
-    # N.B.: The package script requires Python 3.
-    python = sys.executable if PY3 else ensure_python_interpreter(PY38)
-
-    package_script = os.path.join(pex_project_dir, "scripts", "package.py")
     pex_pex = os.path.join(str(tmpdir), "pex")
-    subprocess.check_call(args=[python, package_script, "--pex-output-file", pex_pex])
+    package_script = os.path.join(pex_project_dir, "scripts", "create-packages.py")
+    run_pex_command(
+        args=[
+            "toml",
+            pex_project_dir,
+            "--",
+            package_script,
+            "-v",
+            "--pex-output-file",
+            pex_pex,
+        ],
+        # The package script requires Python>=3.8.
+        python=(
+            sys.executable if sys.version_info[:2] >= (3, 8) else ensure_python_interpreter(PY310)
+        ),
+    ).assert_success()
 
     lock = os.path.join(str(tmpdir), "lock.json")
     subprocess.check_call(

tests/integration/test_issue_1995.py

@@ -8,7 +8,7 @@
 import pytest
 
 from pex.typing import TYPE_CHECKING
-from testing import IS_LINUX, PY38, ensure_python_interpreter, make_env, run_pex_command
+from testing import IS_LINUX, PY310, ensure_python_interpreter, make_env, run_pex_command
 
 if TYPE_CHECKING:
     from typing import Any
@@ -21,7 +21,7 @@ def test_packaging(
 ):
     # type: (...) -> None
     pex = os.path.join(str(tmpdir), "pex.pex")
-    package_script = os.path.join(pex_project_dir, "scripts", "package.py")
+    package_script = os.path.join(pex_project_dir, "scripts", "create-packages.py")
     run_pex_command(
         args=[
             "toml",
@@ -32,8 +32,10 @@ def test_packaging(
             "--pex-output-file",
             pex,
         ],
-        # The package script requires Python 3.
-        python=sys.executable if sys.version_info[0] >= 3 else ensure_python_interpreter(PY38),
+        # The package script requires Python>=3.8.
+        python=(
+            sys.executable if sys.version_info[:2] >= (3, 8) else ensure_python_interpreter(PY310)
+        ),
     ).assert_success()
     assert os.path.exists(pex), "Expected {pex} to be created by {package_script}.".format(
         pex=pex, package_script=package_script