From ea9cdf8ded4c5ba9814b714430b30c67e5254c3c Mon Sep 17 00:00:00 2001 From: Pradyun Gedam Date: Thu, 14 Jul 2022 13:41:58 +0100 Subject: [PATCH] Speed up isolated environment creation Instead of creating a zip file from the current pip's sources, add it to the build environment's interpreter import system using `sys.meta_path`. --- src/pip/_internal/build_env.py | 44 ++++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/src/pip/_internal/build_env.py b/src/pip/_internal/build_env.py index ccf2b4bc472..8c9380331dd 100644 --- a/src/pip/_internal/build_env.py +++ b/src/pip/_internal/build_env.py @@ -7,7 +7,6 @@ import pathlib import sys import textwrap -import zipfile from collections import OrderedDict from sysconfig import get_paths from types import TracebackType @@ -29,6 +28,29 @@ logger = logging.getLogger(__name__) +PIP_RUNNER = """ +import importlib.util +import os +import runpy +import sys + + +class PipImportRedirectingFinder: + + @classmethod + def find_spec(cls, fullname, path=None, target=None): + if not fullname.startswith("pip."): + return None + + # Import pip from the current source directory + location = os.path.join({source!r}, *fullname.split(".")) + return importlib.util.spec_from_file_location(fullname, location) + + +sys.meta_path.append(PipImportRedirectingFinder()) +runpy.run_module("pip", run_name="__main__") +""" + class _Prefix: def __init__(self, path: str) -> None: @@ -42,10 +64,10 @@ def __init__(self, path: str) -> None: @contextlib.contextmanager -def _create_standalone_pip() -> Generator[str, None, None]: - """Create a "standalone pip" zip file. +def _create_runnable_pip() -> Generator[str, None, None]: + """Create a "pip runner" file. - The zip file's content is identical to the currently-running pip. + The runner file ensures that import for pip happen using the currently-running pip. It will be used to install requirements into the build environment. """ source = pathlib.Path(pip_location).resolve().parent @@ -57,14 +79,10 @@ def _create_standalone_pip() -> Generator[str, None, None]: return with TempDirectory(kind="standalone-pip") as tmp_dir: - pip_zip = os.path.join(tmp_dir.path, "__env_pip__.zip") - kwargs = {} - if sys.version_info >= (3, 8): - kwargs["strict_timestamps"] = False - with zipfile.ZipFile(pip_zip, "w", **kwargs) as zf: - for child in source.rglob("*"): - zf.write(child, child.relative_to(source.parent).as_posix()) - yield os.path.join(pip_zip, "pip") + pip_runner = os.path.join(tmp_dir.path, "__pip-runner__.py") + with open(pip_runner, "w") as f: + f.write(PIP_RUNNER.format(source=os.fsdecode(source))) + yield pip_runner class BuildEnvironment: @@ -206,7 +224,7 @@ def install_requirements( if not requirements: return with contextlib.ExitStack() as ctx: - pip_runnable = ctx.enter_context(_create_standalone_pip()) + pip_runnable = ctx.enter_context(_create_runnable_pip()) self._install_requirements( pip_runnable, finder,