Introduce pex3 venv inspect. (#2135)

This command allows inspecting venvs created by Pex as well as those
created by other tools.

Work towards #1752 and #2110

pex/cli/commands/__init__.py

@@ -4,6 +4,7 @@
 from pex.cli.command import BuildTimeCommand
 from pex.cli.commands.interpreter import Interpreter
 from pex.cli.commands.lock import Lock
+from pex.cli.commands.venv import Venv
 from pex.typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
@@ -12,4 +13,4 @@
 
 def all_commands():
     # type: () -> Iterable[Type[BuildTimeCommand]]
-    return Interpreter, Lock
+    return Interpreter, Lock, Venv

pex/cli/commands/venv.py

@@ -0,0 +1,103 @@
+# Copyright 2023 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+import os.path
+from argparse import ArgumentParser, _ActionsContainer
+
+from pex.cli.command import BuildTimeCommand
+from pex.commands.command import JsonMixin, OutputMixin
+from pex.common import is_script
+from pex.pex_info import PexInfo
+from pex.result import Error, Ok, Result
+from pex.typing import TYPE_CHECKING
+from pex.venv.virtualenv import Virtualenv
+
+if TYPE_CHECKING:
+    from typing import Any, Dict
+
+
+class Venv(OutputMixin, JsonMixin, BuildTimeCommand):
+    @classmethod
+    def _add_inspect_arguments(cls, parser):
+        # type: (_ActionsContainer) -> None
+        parser.add_argument(
+            "venv",
+            help="The path of either the venv directory or its Python interpreter.",
+        )
+        cls.add_output_option(parser, entity="venv information")
+        cls.add_json_options(parser, entity="venv information")
+
+    @classmethod
+    def add_extra_arguments(
+        cls,
+        parser,  # type: ArgumentParser
+    ):
+        # type: (...) -> None
+        subcommands = cls.create_subcommands(
+            parser,
+            description="Interact with virtual environments via the following subcommands.",
+        )
+        with subcommands.parser(
+            name="inspect",
+            help="Inspect an existing venv.",
+            func=cls._inspect,
+            include_verbosity=False,
+        ) as inspect_parser:
+            cls._add_inspect_arguments(inspect_parser)
+
+    def _inspect(self):
+        # type: () -> Result
+
+        venv = self.options.venv
+        if not os.path.exists(venv):
+            return Error("The given venv path of {venv} does not exist.".format(venv=venv))
+
+        if os.path.isdir(venv):
+            virtualenv = Virtualenv(os.path.normpath(venv))
+            if not virtualenv.interpreter.is_venv:
+                return Error("{venv} is not a venv.".format(venv=venv))
+        else:
+            maybe_venv = Virtualenv.enclosing(os.path.normpath(venv))
+            if not maybe_venv:
+                return Error("{python} is not an venv interpreter.".format(python=venv))
+            virtualenv = maybe_venv
+
+        try:
+            pex = PexInfo.from_pex(virtualenv.venv_dir)
+            is_pex = True
+            pex_version = pex.build_properties.get("pex_version")
+        except (IOError, OSError, ValueError):
+            is_pex = False
+            pex_version = None
+
+        venv_info = dict(
+            venv_dir=virtualenv.venv_dir,
+            provenance=dict(
+                created_by=virtualenv.created_by,
+                is_pex=is_pex,
+                pex_version=pex_version,
+            ),
+            include_system_site_packages=virtualenv.include_system_site_packages,
+            interpreter=dict(
+                binary=virtualenv.interpreter.binary,
+                base_binary=virtualenv.interpreter.resolve_base_interpreter().binary,
+                version=virtualenv.interpreter.identity.version_str,
+                sys_path=virtualenv.sys_path,
+            ),
+            script_dir=virtualenv.bin_dir,
+            scripts=sorted(
+                os.path.relpath(exe, virtualenv.bin_dir)
+                for exe in virtualenv.iter_executables()
+                if is_script(exe)
+            ),
+            site_packages=virtualenv.site_packages_dir,
+            distributions=sorted(
+                str(dist.as_requirement()) for dist in virtualenv.iter_distributions()
+            ),
+        )  # type: Dict[str, Any]
+
+        with self.output(self.options) as out:
+            self.dump_json(self.options, venv_info, out)
+            out.write("\n")
+
+        return Ok()

pex/dist_metadata.py

@@ -780,6 +780,7 @@ def find_distribution(
 def find_distributions(search_path=None):
     # type: (Optional[Iterable[str]]) -> Iterator[Distribution]
 
+    seen = set()
     for location in search_path or sys.path:
         if not os.path.isdir(location):
             continue
@@ -790,7 +791,10 @@ def find_distributions(search_path=None):
                 for path in glob.glob(os.path.join(location, "*.dist-info/METADATA"))
             ],
         ):
-            metadata_path = os.path.join(location, metadata_file.path)
+            metadata_path = os.path.realpath(os.path.join(location, metadata_file.path))
+            if metadata_path in seen:
+                continue
+            seen.add(metadata_path)
             with open(metadata_path, "rb") as fp:
                 pkg_info = _parse_message(fp.read())
                 yield Distribution(location=location, metadata=DistMetadata.load(pkg_info))

pex/interpreter.py

@@ -449,6 +449,129 @@ def __hash__(self):
         return hash(self._tup())
 
 
+class PyVenvCfg(object):
+    """Represents a pyvenv.cfg file.
+
+    See: https://www.python.org/dev/peps/pep-0405/#specification
+    """
+
+    class Error(ValueError):
+        """Indicates a malformed pyvenv.cfg file."""
+
+    @classmethod
+    def parse(cls, path):
+        # type: (str) -> PyVenvCfg
+        """Attempt to parse `path` as a pyvenv.cfg file.
+
+        :param path: The path of putative pyvenv.cfg file.
+        :raises: :class:`PyVenvCfg.Error` if the given `path` doesn't contain a pyvenv.cfg home key.
+        """
+        # See: https://www.python.org/dev/peps/pep-0405/#specification
+        config = {}
+        with open(path) as fp:
+            for line in fp:
+                raw_name, delimiter, raw_value = line.partition("=")
+                if delimiter != "=":
+                    continue
+                config[raw_name.strip()] = raw_value.strip()
+        if "home" not in config:
+            raise cls.Error("No home config key in {pyvenv_cfg}.".format(pyvenv_cfg=path))
+        return cls(path, **config)
+
+    @classmethod
+    def _get_pyvenv_cfg(cls, path):
+        # type: (str) -> Optional[PyVenvCfg]
+        # See: https://www.python.org/dev/peps/pep-0405/#specification
+        pyvenv_cfg_path = os.path.join(path, "pyvenv.cfg")
+        if os.path.isfile(pyvenv_cfg_path):
+            try:
+                return cls.parse(pyvenv_cfg_path)
+            except cls.Error:
+                pass
+        return None
+
+    @classmethod
+    def find(cls, python_binary):
+        # type: (str) -> Optional[PyVenvCfg]
+        """Attempt to find a pyvenv.cfg file identifying a virtualenv enclosing a Python binary.
+
+        :param python_binary: The path of a Python binary (can be a symlink).
+        """
+        # A pyvenv is identified by a pyvenv.cfg file with a home key in one of the two following
+        # directory layouts:
+        #
+        # 1. <venv dir>/
+        #      bin/
+        #        pyvenv.cfg
+        #        python*
+        #
+        # 2. <venv dir>/
+        #      pyvenv.cfg
+        #      bin/
+        #        python*
+        #
+        # In practice, we see layout 2 in the wild, but layout 1 is also allowed by the spec.
+        #
+        # See: # See: https://www.python.org/dev/peps/pep-0405/#specification
+        maybe_venv_bin_dir = os.path.dirname(python_binary)
+        pyvenv_cfg = cls._get_pyvenv_cfg(maybe_venv_bin_dir)
+        if not pyvenv_cfg:
+            maybe_venv_dir = os.path.dirname(maybe_venv_bin_dir)
+            pyvenv_cfg = cls._get_pyvenv_cfg(maybe_venv_dir)
+        return pyvenv_cfg
+
+    def __init__(
+        self,
+        path,  # type: str
+        **config  # type: str
+    ):
+        # type: (...) -> None
+        self._path = path
+        self._config = config
+
+    @property
+    def path(self):
+        # type: () -> str
+        return self._path
+
+    @property
+    def home(self):
+        # type: () -> str
+        return self._config["home"]
+
+    @overload
+    def config(
+        self,
+        key,  # type: str
+        default=None,  # type: None
+    ):
+        # type: (...) -> Optional[str]
+        pass
+
+    @overload
+    def config(
+        self,
+        key,  # type: str
+        default,  # type: str
+    ):
+        # type: (...) -> str
+        pass
+
+    def config(
+        self,
+        key,  # type: str
+        default=None,  # type: Optional[str]
+    ):
+        # type: (...) -> Optional[str]
+        return self._config.get(key, default)
+
+    @property
+    def include_system_site_packages(self):
+        # type: () -> Optional[bool]
+        value = self.config("include-system-site-packages")
+        return value.lower() == "true" if value else None
+
+
 class PythonInterpreter(object):
     _REGEXEN = (
         # NB: OSX ships python binaries named Python with a capital-P; so we allow for this.
@@ -493,45 +616,6 @@ def _cleared_memory_cache(cls):
         finally:
             cls._PYTHON_INTERPRETER_BY_NORMALIZED_PATH = _cache
 
-    @staticmethod
-    def _get_pyvenv_cfg(path):
-        # type: (str) -> Optional[str]
-        # See: https://www.python.org/dev/peps/pep-0405/#specification
-        pyvenv_cfg_path = os.path.join(path, "pyvenv.cfg")
-        if os.path.isfile(pyvenv_cfg_path):
-            with open(pyvenv_cfg_path) as fp:
-                for line in fp:
-                    name, _, value = line.partition("=")
-                    if name.strip() == "home":
-                        return pyvenv_cfg_path
-        return None
-
-    @classmethod
-    def _find_pyvenv_cfg(cls, maybe_venv_python_binary):
-        # type: (str) -> Optional[str]
-        # A pyvenv is identified by a pyvenv.cfg file with a home key in one of the two following
-        # directory layouts:
-        #
-        # 1. <venv dir>/
-        #      bin/
-        #        pyvenv.cfg
-        #        python*
-        #
-        # 2. <venv dir>/
-        #      pyvenv.cfg
-        #      bin/
-        #        python*
-        #
-        # In practice, we see layout 2 in the wild, but layout 1 is also allowed by the spec.
-        #
-        # See: # See: https://www.python.org/dev/peps/pep-0405/#specification
-        maybe_venv_bin_dir = os.path.dirname(maybe_venv_python_binary)
-        pyvenv_cfg = cls._get_pyvenv_cfg(maybe_venv_bin_dir)
-        if not pyvenv_cfg:
-            maybe_venv_dir = os.path.dirname(maybe_venv_bin_dir)
-            pyvenv_cfg = cls._get_pyvenv_cfg(maybe_venv_dir)
-        return pyvenv_cfg
-
     @classmethod
     def _resolve_pyvenv_canonical_python_binary(
         cls,
@@ -542,7 +626,7 @@ def _resolve_pyvenv_canonical_python_binary(
         if not os.path.islink(maybe_venv_python_binary):
             return None
 
-        pyvenv_cfg = cls._find_pyvenv_cfg(maybe_venv_python_binary)
+        pyvenv_cfg = PyVenvCfg.find(maybe_venv_python_binary)
         if pyvenv_cfg is None:
             return None