diff --git a/src/pip/_internal/commands/download.py b/src/pip/_internal/commands/download.py index 0861d9e67b2..ac16ac79a03 100644 --- a/src/pip/_internal/commands/download.py +++ b/src/pip/_internal/commands/download.py @@ -2,11 +2,13 @@ import logging import os +import shutil from pip._internal.cli import cmdoptions from pip._internal.cli.cmdoptions import make_target_python from pip._internal.cli.req_command import RequirementCommand, with_cleanup from pip._internal.cli.status_codes import SUCCESS +from pip._internal.network.download import BatchDownloader from pip._internal.req.req_tracker import get_requirement_tracker from pip._internal.utils.misc import ensure_dir, normalize_path, write_output from pip._internal.utils.temp_dir import TempDirectory @@ -134,6 +136,24 @@ def run(self, options, args): reqs, check_supported_wheels=True ) + # Download any requirements which were only partially downloaded with + # --use-feature=fast-deps. + reqs_to_fully_download = [ + r for r in requirement_set.requirements.values() + if r.needs_more_preparation + ] + links_from_newly_downloaded = [] + for partially_downloaded_req in reqs_to_fully_download: + assert partially_downloaded_req.link + links_from_newly_downloaded.append(partially_downloaded_req.link) + # Let's download to a temporary directory. + tmpdir = TempDirectory(kind="unpack", globally_managed=True).path + batch_download = BatchDownloader(session, + progress_bar=options.progress_bar) + download_iterable = batch_download(links_from_newly_downloaded, tmpdir) + for _link, (filepath, _) in download_iterable: + shutil.copy(filepath, options.download_dir) + downloaded = [] # type: List[str] for req in requirement_set.requirements.values(): if not req.editable and req.satisfied_by is None: diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index acff7ffa1f7..f01a4dcd48a 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -506,24 +506,23 @@ def prepare_linked_requirement(self, req, parallel_builds=False): def prepare_linked_requirements_more(self, reqs, parallel_builds=False): # type: (Iterable[InstallRequirement], bool) -> None - """Prepare a linked requirement more, if needed.""" + """Prepare linked requirements more, if needed.""" reqs = [req for req in reqs if req.needs_more_preparation] - links = [] # type: List[Link] for req in reqs: + # Determine if any of these requirements were already downloaded. download_dir = self._get_download_dir(req.link) if download_dir is not None: hashes = self._get_linked_req_hashes(req) file_path = _check_download_dir(req.link, download_dir, hashes) - if download_dir is None or file_path is None: - links.append(req.link) - else: - self._downloaded[req.link.url] = file_path, None - - # Let's download to a temporary directory. - tmpdir = TempDirectory(kind="unpack", globally_managed=True).path - self._downloaded.update(self._batch_download(links, tmpdir)) + if file_path is not None: + self._downloaded[req.link.url] = file_path, None + req.needs_more_preparation = False + + # Prepare requirements we found were already downloaded for some + # reason. The rest will be downloaded outside of the resolver. for req in reqs: - self._prepare_linked_requirement(req, parallel_builds) + if not req.needs_more_preparation: + self._prepare_linked_requirement(req, parallel_builds) def _prepare_linked_requirement(self, req, parallel_builds): # type: (InstallRequirement, bool) -> Distribution