Add --site-packages-copies for external venvs. (#2470)

When creating a venv external to the `PEX_ROOT` via either
`pex3 venv create ...` or `PEX_TOOLS=1 ./my.pex venv ...` you can now
specify `--site-packages-copies` to ensure all code populated in the
venv is isolated from the `PEX_ROOT` cache. Although this takes more
space on disk, it may make sense to use when you expect the venv might
be tampered with or used for experimentation that might alter its files;
otherwise contaminating the `PEX_ROOT` cache.

Fixes #2313

pex/bin/pex.py

@@ -23,7 +23,7 @@
     global_environment,
     register_global_arguments,
 )
-from pex.common import die, is_pyc_dir, is_pyc_file, safe_mkdtemp
+from pex.common import CopyMode, die, is_pyc_dir, is_pyc_file, safe_mkdtemp
 from pex.dependency_configuration import DependencyConfiguration
 from pex.dependency_manager import DependencyManager
 from pex.dist_metadata import Requirement
@@ -38,7 +38,7 @@
 from pex.pep_723 import ScriptMetadata
 from pex.pex import PEX
 from pex.pex_bootstrapper import ensure_venv
-from pex.pex_builder import Check, CopyMode, PEXBuilder
+from pex.pex_builder import Check, PEXBuilder
 from pex.pex_info import PexInfo
 from pex.resolve import (
     project,

pex/cli/commands/venv.py

@@ -11,7 +11,7 @@
 from pex import pex_warnings
 from pex.cli.command import BuildTimeCommand
 from pex.commands.command import JsonMixin, OutputMixin
-from pex.common import DETERMINISTIC_DATETIME, is_script, open_zip, pluralize
+from pex.common import DETERMINISTIC_DATETIME, CopyMode, is_script, open_zip, pluralize
 from pex.dist_metadata import Distribution
 from pex.enum import Enum
 from pex.executor import Executor
@@ -311,15 +311,23 @@ def _create(self):
                     venv=venv,
                     distributions=distributions,
                     provenance=provenance,
-                    symlink=False,
+                    copy_mode=(
+                        CopyMode.COPY
+                        if installer_configuration.site_packages_copies
+                        else CopyMode.LINK
+                    ),
                     hermetic_scripts=hermetic_scripts,
                 )
             else:
                 installer.populate_flat_distributions(
                     dest_dir=dest_dir,
                     distributions=distributions,
                     provenance=provenance,
-                    symlink=False,
+                    copy_mode=(
+                        CopyMode.COPY
+                        if installer_configuration.site_packages_copies
+                        else CopyMode.LINK
+                    ),
                 )
             source = (
                 "PEX at {pex}".format(pex=pex.path())
@@ -388,7 +396,9 @@ def _install_from_pex(
                 venv=venv,
                 distributions=distributions,
                 provenance=provenance,
-                symlink=False,
+                copy_mode=(
+                    CopyMode.COPY if installer_configuration.site_packages_copies else CopyMode.LINK
+                ),
                 hermetic_scripts=hermetic_scripts,
                 top_level_source_packages=top_level_source_packages,
             )
@@ -397,7 +407,9 @@ def _install_from_pex(
                 dest_dir=dest_dir,
                 distributions=distributions,
                 provenance=provenance,
-                symlink=False,
+                copy_mode=(
+                    CopyMode.COPY if installer_configuration.site_packages_copies else CopyMode.LINK
+                ),
             )
 
     if installer_configuration.scope in (InstallScope.ALL, InstallScope.SOURCE_ONLY):

pex/common.py

@@ -20,6 +20,7 @@
 from datetime import datetime
 from uuid import uuid4
 
+from pex.enum import Enum
 from pex.typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
@@ -767,11 +768,20 @@ def relative_symlink(
     os.symlink(rel_src, dst)
 
 
+class CopyMode(Enum["CopyMode.Value"]):
+    class Value(Enum.Value):
+        pass
+
+    COPY = Value("copy")
+    LINK = Value("link")
+    SYMLINK = Value("symlink")
+
+
 def iter_copytree(
     src,  # type: Text
     dst,  # type: Text
     exclude=(),  # type: Container[Text]
-    symlink=False,  # type: bool
+    copy_mode=CopyMode.LINK,  # type: CopyMode.Value
 ):
     # type: (...) -> Iterator[Tuple[Text, Text]]
     """Copies the directory tree rooted at `src` to `dst` yielding a tuple for each copied file.
@@ -784,11 +794,11 @@ def iter_copytree(
     :param src: The source directory tree to copy.
     :param dst: The destination location to copy the source tree to.
     :param exclude: Names (basenames) of files and directories to exclude from copying.
-    :param symlink: Whether to use symlinks instead of copies (or hard links).
+    :param copy_mode: How to copy files.
     :return: An iterator over tuples identifying the copied files of the form `(src, dst)`.
     """
     safe_mkdir(dst)
-    link = True
+    link = copy_mode is CopyMode.LINK
     for root, dirs, files in os.walk(src, topdown=True, followlinks=True):
         if src == root:
             dirs[:] = [d for d in dirs if d not in exclude]
@@ -802,7 +812,7 @@ def iter_copytree(
             if not is_dir:
                 yield src_entry, dst_entry
             try:
-                if symlink:
+                if copy_mode is CopyMode.SYMLINK:
                     relative_symlink(src_entry, dst_entry)
                 elif is_dir:
                     os.mkdir(dst_entry)
@@ -823,6 +833,6 @@ def iter_copytree(
                 if e.errno != errno.EEXIST:
                     raise e
 
-        if symlink:
+        if copy_mode is CopyMode.SYMLINK:
             # Once we've symlinked the top-level directories and files, we've "copied" everything.
             return

pex/pep_376.py

@@ -14,7 +14,7 @@
 from fileinput import FileInput
 
 from pex import hashing
-from pex.common import is_pyc_dir, is_pyc_file, safe_mkdir, safe_open
+from pex.common import CopyMode, is_pyc_dir, is_pyc_file, safe_mkdir, safe_open
 from pex.interpreter import PythonInterpreter
 from pex.typing import TYPE_CHECKING, cast
 from pex.util import CacheHelper
@@ -280,7 +280,7 @@ def _create_record(
     def reinstall_flat(
         self,
         target_dir,  # type: str
-        symlink=False,  # type: bool
+        copy_mode=CopyMode.LINK,  # type: CopyMode.Value
     ):
         # type: (...) -> Iterator[Tuple[Text, Text]]
         """Re-installs the installed wheel in a flat target directory.
@@ -296,8 +296,8 @@ def reinstall_flat(
         """
         installed_files = [InstalledFile(self.record_relpath)]
         for src, dst in itertools.chain(
-            self._reinstall_stash(dest_dir=target_dir),
-            self._reinstall_site_packages(target_dir, symlink=symlink),
+            self._reinstall_stash(dest_dir=target_dir, link=copy_mode is not CopyMode.COPY),
+            self._reinstall_site_packages(target_dir, copy_mode=copy_mode),
         ):
             installed_files.append(self.create_installed_file(path=dst, dest_dir=target_dir))
             yield src, dst
@@ -307,7 +307,7 @@ def reinstall_flat(
     def reinstall_venv(
         self,
         venv,  # type: Virtualenv
-        symlink=False,  # type: bool
+        copy_mode=CopyMode.LINK,  # type: CopyMode.Value
         rel_extra_path=None,  # type: Optional[str]
     ):
         # type: (...) -> Iterator[Tuple[Text, Text]]
@@ -330,8 +330,12 @@ def reinstall_venv(
 
         installed_files = [InstalledFile(self.record_relpath)]
         for src, dst in itertools.chain(
-            self._reinstall_stash(dest_dir=venv.venv_dir, interpreter=venv.interpreter),
-            self._reinstall_site_packages(site_packages_dir, symlink=symlink),
+            self._reinstall_stash(
+                dest_dir=venv.venv_dir,
+                interpreter=venv.interpreter,
+                link=copy_mode is not CopyMode.COPY,
+            ),
+            self._reinstall_site_packages(site_packages_dir, copy_mode=copy_mode),
         ):
             installed_files.append(self.create_installed_file(path=dst, dest_dir=site_packages_dir))
             yield src, dst
@@ -342,10 +346,10 @@ def _reinstall_stash(
         self,
         dest_dir,  # type: str
         interpreter=None,  # type: Optional[PythonInterpreter]
+        link=True,  # type: bool
     ):
         # type: (...) -> Iterator[Tuple[Text, Text]]
 
-        link = True
         stash_abs_path = os.path.join(self.prefix_dir, self.stash_dir)
         for root, dirs, files in os.walk(stash_abs_path, topdown=True, followlinks=True):
             dir_created = False
@@ -380,11 +384,11 @@ def _reinstall_stash(
     def _reinstall_site_packages(
         self,
         site_packages_dir,  # type: str
-        symlink=False,  # type: bool
+        copy_mode=CopyMode.LINK,  # type: CopyMode.Value
     ):
         # type: (...) -> Iterator[Tuple[Text, Text]]
 
-        link = True
+        link = copy_mode is CopyMode.LINK
         for root, dirs, files in os.walk(self.prefix_dir, topdown=True, followlinks=True):
             if root == self.prefix_dir:
                 dirs[:] = [d for d in dirs if not is_pyc_dir(d) and d != self.stash_dir]
@@ -401,7 +405,7 @@ def _reinstall_site_packages(
                     site_packages_dir, os.path.relpath(src_entry, self.prefix_dir)
                 )
                 try:
-                    if symlink and not (
+                    if copy_mode is CopyMode.SYMLINK and not (
                         src_entry.endswith(".dist-info") and os.path.isdir(src_entry)
                     ):
                         dst_parent = os.path.dirname(dst_entry)

pex/pex_bootstrapper.py

@@ -9,7 +9,7 @@
 
 from pex import pex_warnings
 from pex.atomic_directory import atomic_directory
-from pex.common import die, pluralize
+from pex.common import CopyMode, die, pluralize
 from pex.environment import ResolveError
 from pex.inherit_path import InheritPath
 from pex.interpreter import PythonInterpreter
@@ -555,7 +555,11 @@ def ensure_venv(
                     # so we'll not have a stable base there to symlink from. As such, always copy
                     # for loose PEXes to ensure the PEX_ROOT venv is stable in the face of
                     # modification of the source loose PEX.
-                    symlink = pex.layout != Layout.LOOSE and not pex_info.venv_site_packages_copies
+                    copy_mode = (
+                        CopyMode.SYMLINK
+                        if (pex.layout != Layout.LOOSE and not pex_info.venv_site_packages_copies)
+                        else CopyMode.LINK
+                    )
 
                     shebang = installer.populate_venv_from_pex(
                         virtualenv,
@@ -568,7 +572,7 @@ def ensure_venv(
                             os.path.basename(virtualenv.interpreter.binary),
                         ),
                         collisions_ok=collisions_ok,
-                        symlink=symlink,
+                        copy_mode=copy_mode,
                         hermetic_scripts=pex_info.venv_hermetic_scripts,
                     )
 

pex/pex_builder.py

@@ -15,6 +15,7 @@
 from pex.atomic_directory import atomic_directory
 from pex.common import (
     Chroot,
+    CopyMode,
     chmod_plus_x,
     deterministic_walk,
     is_pyc_file,
@@ -51,15 +52,6 @@
 _ABS_PEX_PACKAGE_DIR = os.path.dirname(os.path.abspath(__file__))
 
 
-class CopyMode(Enum["CopyMode.Value"]):
-    class Value(Enum.Value):
-        pass
-
-    COPY = Value("copy")
-    LINK = Value("link")
-    SYMLINK = Value("symlink")
-
-
 class InvalidZipAppError(Exception):
     pass
 

pex/tools/commands/venv.py

@@ -9,7 +9,7 @@
 from argparse import ArgumentParser
 
 from pex import pex_warnings
-from pex.common import safe_delete, safe_rmtree
+from pex.common import CopyMode, safe_delete, safe_rmtree
 from pex.enum import Enum
 from pex.executor import Executor
 from pex.pex import PEX
@@ -143,7 +143,9 @@ def run(self, pex):
             pex,
             bin_path=installer_configuration.bin_path,
             collisions_ok=installer_configuration.collisions_ok,
-            symlink=False,
+            copy_mode=(
+                CopyMode.COPY if installer_configuration.site_packages_copies else CopyMode.LINK
+            ),
             scope=installer_configuration.scope,
             hermetic_scripts=installer_configuration.hermetic_scripts,
         )