try to build wheels for directories/vcs

news/4501.feature

@@ -0,0 +1,3 @@
+Installing from a local directory or a VCS URL now builds a wheel to install,
+rather than running ``setup.py install``. Wheels from these sources are not
+cached.

src/pip/_internal/cache.py

@@ -12,6 +12,7 @@
 from pip._internal.compat import expanduser
 from pip._internal.download import path_to_url
 from pip._internal.wheel import InvalidWheelFilename, Wheel
+from pip._internal.utils import temp_dir
 
 logger = logging.getLogger(__name__)
 
@@ -32,6 +33,9 @@ def __init__(self, cache_dir, format_control, allowed_formats):
         self.cache_dir = expanduser(cache_dir) if cache_dir else None
         self.format_control = format_control
         self.allowed_formats = allowed_formats
+        # Ephemeral cache: store wheels just for this run
+        self._ephem_cache_dir = temp_dir.TempDirectory(kind="ephem-cache")
+        self._ephem_cache_dir.create()
 
         _valid_formats = {"source", "binary"}
         assert self.allowed_formats.union(_valid_formats) == _valid_formats
@@ -102,6 +106,9 @@ def _link_for_candidate(self, link, candidate):
 
         return index.Link(path_to_url(path))
 
+    def cleanup(self):
+        """Remove the ephermal caches created temporarily to build wheels"""
+        self._ephem_cache_dir.cleanup()
 
 class WheelCache(Cache):
     """A cache of wheels for future installs.
@@ -110,7 +117,7 @@ class WheelCache(Cache):
     def __init__(self, cache_dir, format_control):
         super(WheelCache, self).__init__(cache_dir, format_control, {"binary"})
 
-    def get_path_for_link(self, link):
+    def get_path_for_link(self, link, ephem=False):
         """Return a directory to store cached wheels for link
 
         Because there are M wheels for any one sdist, we provide a directory
@@ -127,9 +134,14 @@ def get_path_for_link(self, link):
         """
         parts = self._get_cache_path_parts(link)
 
+        if not ephem:
+            cache_dir = self.cache_dir
+        else:
+            cache_dir = self._ephem_cache_dir.path
+
         # Inside of the base location for cached wheels, expand our parts and
         # join them all together.
-        return os.path.join(self.cache_dir, "wheels", *parts)
+        return os.path.join(cache_dir, "wheels", *parts)
 
     def get(self, link, package_name):
         candidates = []

src/pip/_internal/commands/freeze.py

@@ -88,5 +88,8 @@ def run(self, options, args):
             skip=skip,
             exclude_editable=options.exclude_editable)
 
-        for line in freeze(**freeze_kwargs):
-            sys.stdout.write(line + '\n')
+        try:
+            for line in freeze(**freeze_kwargs):
+                sys.stdout.write(line + '\n')
+        finally:
+            wheel_cache.cleanup()

src/pip/_internal/commands/install.py

@@ -247,18 +247,19 @@ def run(self, options, args):
                     use_user_site=options.use_user_site,
                 )
 
-                self.populate_requirement_set(
-                    requirement_set, args, options, finder, session, self.name,
-                    wheel_cache
-                )
-                preparer = RequirementPreparer(
-                    build_dir=directory.path,
-                    src_dir=options.src_dir,
-                    download_dir=None,
-                    wheel_download_dir=None,
-                    progress_bar=options.progress_bar,
-                )
                 try:
+                    self.populate_requirement_set(
+                        requirement_set, args, options, finder, session,
+                        self.name, wheel_cache
+                    )
+                    preparer = RequirementPreparer(
+                        build_dir=directory.path,
+                        src_dir=options.src_dir,
+                        download_dir=None,
+                        wheel_download_dir=None,
+                        progress_bar=options.progress_bar,
+                    )
+
                     resolver = Resolver(
                         preparer=preparer,
                         finder=finder,
@@ -350,6 +351,7 @@ def run(self, options, args):
                     # Clean up
                     if not options.no_clean:
                         requirement_set.cleanup_files()
+                        wheel_cache.cleanup()
 
         if options.target_dir:
             self._handle_target_dir(

src/pip/_internal/commands/wheel.py

@@ -146,35 +146,35 @@ def run(self, options, args):
                     require_hashes=options.require_hashes,
                 )
 
-                self.populate_requirement_set(
-                    requirement_set, args, options, finder, session, self.name,
-                    wheel_cache
-                )
+                try:
+                    self.populate_requirement_set(
+                        requirement_set, args, options, finder, session,
+                        self.name, wheel_cache
+                    )
 
-                preparer = RequirementPreparer(
-                    build_dir=directory.path,
-                    src_dir=options.src_dir,
-                    download_dir=None,
-                    wheel_download_dir=options.wheel_dir,
-                    progress_bar=options.progress_bar,
-                )
+                    preparer = RequirementPreparer(
+                        build_dir=directory.path,
+                        src_dir=options.src_dir,
+                        download_dir=None,
+                        wheel_download_dir=options.wheel_dir,
+                        progress_bar=options.progress_bar,
+                    )
 
-                resolver = Resolver(
-                    preparer=preparer,
-                    finder=finder,
-                    session=session,
-                    wheel_cache=wheel_cache,
-                    use_user_site=False,
-                    upgrade_strategy="to-satisfy-only",
-                    force_reinstall=False,
-                    ignore_dependencies=options.ignore_dependencies,
-                    ignore_requires_python=options.ignore_requires_python,
-                    ignore_installed=True,
-                    isolated=options.isolated_mode,
-                )
-                resolver.resolve(requirement_set)
+                    resolver = Resolver(
+                        preparer=preparer,
+                        finder=finder,
+                        session=session,
+                        wheel_cache=wheel_cache,
+                        use_user_site=False,
+                        upgrade_strategy="to-satisfy-only",
+                        force_reinstall=False,
+                        ignore_dependencies=options.ignore_dependencies,
+                        ignore_requires_python=options.ignore_requires_python,
+                        ignore_installed=True,
+                        isolated=options.isolated_mode,
+                    )
+                    resolver.resolve(requirement_set)
 
-                try:
                     # build wheels
                     wb = WheelBuilder(
                         requirement_set,
@@ -196,3 +196,4 @@ def run(self, options, args):
                 finally:
                     if not options.no_clean:
                         requirement_set.cleanup_files()
+                        wheel_cache.cleanup()

src/pip/_internal/req/req_set.py

@@ -16,9 +16,6 @@ def __init__(self,
                  require_hashes=False, target_dir=None, use_user_site=False,
                  pycompile=True):
         """Create a RequirementSet.
-
-        :param wheel_cache: The pip wheel cache, for passing to
-            InstallRequirement.
         """
 
         self.requirements = OrderedDict()

src/pip/_internal/wheel.py

@@ -741,6 +741,7 @@ def build(self, session, autobuilding=False):
 
         buildset = []
         for req in reqset:
+            ephem_cache = False
             if req.constraint:
                 continue
             if req.is_wheel:
@@ -750,41 +751,42 @@ def build(self, session, autobuilding=False):
             elif autobuilding and req.editable:
                 pass
             elif autobuilding and req.link and not req.link.is_artifact:
-                pass
+                # VCS checkout. Build wheel just for this run.
+                ephem_cache = True
             elif autobuilding and not req.source_dir:
                 pass
             else:
                 if autobuilding:
                     link = req.link
                     base, ext = link.splitext()
                     if index.egg_info_matches(base, None, link) is None:
-                        # Doesn't look like a package - don't autobuild a wheel
-                        # because we'll have no way to lookup the result sanely
-                        continue
-                    if "binary" not in index.fmt_ctl_formats(
+                        # E.g. local directory. Build wheel just for this run.
+                        ephem_cache = True
+                    elif "binary" not in index.fmt_ctl_formats(
                             self.finder.format_control,
                             canonicalize_name(req.name)):
                         logger.info(
                             "Skipping bdist_wheel for %s, due to binaries "
                             "being disabled for it.", req.name)
                         continue
-                buildset.append(req)
+                buildset.append((req, ephem_cache))
 
         if not buildset:
             return True
 
         # Build the wheels.
         logger.info(
             'Building wheels for collected packages: %s',
-            ', '.join([req.name for req in buildset]),
+            ', '.join([req.name for (req, _) in buildset]),
         )
         with indent_log():
             build_success, build_failure = [], []
-            for req in buildset:
+            for req, ephem in buildset:
                 python_tag = None
                 if autobuilding:
                     python_tag = pep425tags.implementation_tag
-                    output_dir = self.wheel_cache.get_path_for_link(req.link)
+                    output_dir = self.wheel_cache.get_path_for_link(req.link,
+                                                                    ephem=ephem)
                     try:
                         ensure_dir(output_dir)
                     except OSError as e:

tests/functional/test_install.py

@@ -983,16 +983,15 @@ def test_install_builds_wheels(script, data, common_wheels):
     # and built wheels for upper and wheelbroken
     assert "Running setup.py bdist_wheel for upper" in str(res), str(res)
     assert "Running setup.py bdist_wheel for wheelb" in str(res), str(res)
-    # But not requires_wheel... which is a local dir and thus uncachable.
-    assert "Running setup.py bdist_wheel for requir" not in str(res), str(res)
+    # Wheels are built for local directories, but not cached.
+    assert "Running setup.py bdist_wheel for requir" in str(res), str(res)
     # wheelbroken has to run install
     # into the cache
     assert wheels != [], str(res)
     # and installed from the wheel
     assert "Running setup.py install for upper" not in str(res), str(res)
-    # the local tree can't build a wheel (because we can't assume that every
-    # build will have a suitable unique key to cache on).
-    assert "Running setup.py install for requires-wheel" in str(res), str(res)
+    # Wheels are built for local directories, but not cached.
+    assert "Running setup.py install for requir" not in str(res), str(res)
     # wheelbroken has to run install
     assert "Running setup.py install for wheelb" in str(res), str(res)
     # We want to make sure we used the correct implementation tag
@@ -1016,13 +1015,15 @@ def test_install_no_binary_disables_building_wheels(
     assert expected in str(res), str(res)
     # and built wheels for wheelbroken only
     assert "Running setup.py bdist_wheel for wheelb" in str(res), str(res)
-    # But not requires_wheel... which is a local dir and thus uncachable.
-    assert "Running setup.py bdist_wheel for requir" not in str(res), str(res)
+    # Wheels are built for local directories, but not cached
+    assert "Running setup.py bdist_wheel for requir" in str(res), str(res)
     # Nor upper, which was blacklisted
     assert "Running setup.py bdist_wheel for upper" not in str(res), str(res)
-    # the local tree can't build a wheel (because we can't assume that every
-    # build will have a suitable unique key to cache on).
-    assert "Running setup.py install for requires-wheel" in str(res), str(res)
+    # wheelbroken has to run install
+    # into the cache
+    assert wheels != [], str(res)
+    # Wheels are built for local directories, but not cached
+    assert "Running setup.py install for requir" not in str(res), str(res)
     # And these two fell back to sdist based installed.
     assert "Running setup.py install for wheelb" in str(res), str(res)
     assert "Running setup.py install for upper" in str(res), str(res)

tests/functional/test_install_cleanup.py

@@ -30,7 +30,7 @@ def test_no_clean_option_blocks_cleaning_after_install(script, data):
     build = script.base_path / 'pip-build'
     script.pip(
         'install', '--no-clean', '--no-index', '--build', build,
-        '--find-links=%s' % data.find_links, 'simple',
+        '--find-links=%s' % data.find_links, 'simple', expect_temp=True,
     )
     assert exists(build)
 
@@ -132,7 +132,7 @@ def test_cleanup_prevented_upon_build_dir_exception(script, data):
     result = script.pip(
         'install', '-f', data.find_links, '--no-index', 'simple',
         '--build', build,
-        expect_error=True,
+        expect_error=True, expect_temp=True,
     )
 
     assert result.returncode == PREVIOUS_BUILD_DIR_ERROR

tests/functional/test_install_user.py

@@ -68,24 +68,26 @@ def test_install_subversion_usersite_editable_with_distribute(
         )
         result.assert_installed('INITools', use_user_site=True)
 
+    @pytest.mark.network
     def test_install_curdir_usersite(self, script, virtualenv, data):
         """
         Test installing current directory ('.') into usersite
         """
         virtualenv.system_site_packages = True
+        script.pip("install", "wheel")
         run_from = data.packages.join("FSPkg")
         result = script.pip(
             'install', '-vvv', '--user', curdir,
             cwd=run_from,
             expect_error=False,
         )
         fspkg_folder = script.user_site / 'fspkg'
-        egg_info_folder = (
-            script.user_site / 'FSPkg-0.1.dev0-py%s.egg-info' % pyversion
+        dist_info_folder = (
+            script.user_site / 'FSPkg-0.1.dev0.dist-info'
         )
         assert fspkg_folder in result.files_created, result.stdout
 
-        assert egg_info_folder in result.files_created
+        assert dist_info_folder in result.files_created
 
     def test_install_user_venv_nositepkgs_fails(self, script, data):
         """

tests/functional/test_wheel.py

@@ -193,7 +193,7 @@ def test_pip_wheel_fail_cause_of_previous_build_dir(
     result = script.pip(
         'wheel', '--no-index', '--find-links=%s' % data.find_links,
         '--build', script.venv_path / 'build',
-        'simple==3.0', expect_error=True,
+        'simple==3.0', expect_error=True, expect_temp=True,
     )
 
     # Then I see that the error code is the right one