From d638a9504fd3fdb7c8133a07add6f1c06588f6d9 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Wed, 11 Sep 2024 20:16:51 -0700 Subject: [PATCH] Refactor to a PreResolvedResolver. Also fix up resolve checking for both the Pip resolver and the pre-resolver. --- pex/bin/pex.py | 128 ++------------------ pex/cli/commands/lock.py | 1 + pex/cli/commands/venv.py | 4 +- pex/dependency_manager.py | 10 +- pex/dist_metadata.py | 67 +++++++++-- pex/environment.py | 31 ++--- pex/pep_425.py | 5 +- pex/resolve/configured_resolve.py | 26 ++++ pex/resolve/lock_resolver.py | 4 +- pex/resolve/locked_resolve.py | 51 +------- pex/resolve/lockfile/create.py | 6 +- pex/resolve/pre_resolved_resolver.py | 163 ++++++++++++++++++++++++++ pex/resolve/resolved_requirement.py | 4 +- pex/resolve/resolver_configuration.py | 17 +++ pex/resolve/resolver_options.py | 79 +++++++++++-- pex/resolve/resolvers.py | 91 +++++++++++++- pex/resolver.py | 91 ++------------ pex/targets.py | 39 +++++- tests/integration/test_integration.py | 6 +- tests/integration/test_issue_1179.py | 17 ++- tests/test_dist_metadata.py | 2 +- tests/test_resolver.py | 6 +- 22 files changed, 531 insertions(+), 317 deletions(-) create mode 100644 pex/resolve/pre_resolved_resolver.py diff --git a/pex/bin/pex.py b/pex/bin/pex.py index 6b45221d1..cf6bcb32e 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -8,15 +8,12 @@ from __future__ import absolute_import, print_function -import glob -import hashlib import itertools import json import os import shlex import sys from argparse import Action, ArgumentDefaultsHelpFormatter, ArgumentError, ArgumentParser -from collections import OrderedDict from textwrap import TextWrapper from pex import dependency_configuration, pex_warnings, repl, scie @@ -29,14 +26,12 @@ from pex.common import CopyMode, die, is_pyc_dir, is_pyc_file, safe_mkdtemp from pex.dependency_configuration import DependencyConfiguration from pex.dependency_manager import DependencyManager -from pex.dist_metadata import Distribution, Requirement +from pex.dist_metadata import Requirement from pex.docs.command import serve_html_docs from pex.enum import Enum from pex.fetcher import URLFetcher -from pex.fingerprinted_distribution import FingerprintedDistribution from pex.inherit_path import InheritPath from pex.interpreter_constraints import InterpreterConstraint, InterpreterConstraints -from pex.jobs import iter_map_parallel from pex.layout import Layout, ensure_installed from pex.orderedset import OrderedSet from pex.pep_427 import InstallableType @@ -45,7 +40,6 @@ from pex.pex_bootstrapper import ensure_venv from pex.pex_builder import Check, PEXBuilder from pex.pex_info import PexInfo -from pex.pip.tool import PackageIndexConfiguration from pex.resolve import ( project, requirement_options, @@ -55,21 +49,19 @@ ) from pex.resolve.config import finalize as finalize_resolve_config from pex.resolve.configured_resolve import resolve -from pex.resolve.configured_resolver import ConfiguredResolver from pex.resolve.requirement_configuration import RequirementConfiguration from pex.resolve.resolver_configuration import ( LockRepositoryConfiguration, PexRepositoryConfiguration, + PreResolvedConfiguration, ) from pex.resolve.resolver_options import create_pip_configuration from pex.resolve.resolvers import Unsatisfiable, sorted_requirements -from pex.resolver import BuildAndInstallRequest, BuildRequest, InstallRequest from pex.result import Error, ResultError, catch, try_ from pex.scie import ScieConfiguration from pex.targets import Targets from pex.tracer import TRACER from pex.typing import TYPE_CHECKING, cast -from pex.util import CacheHelper from pex.variables import ENV, Variables from pex.venv.bin_path import BinPath from pex.version import __version__ @@ -145,7 +137,9 @@ def configure_clp_pex_resolution(parser): ), ) - resolver_options.register(group, include_pex_repository=True, include_lock=True) + resolver_options.register( + group, include_pex_repository=True, include_lock=True, include_pre_resolved=True + ) group.add_argument( "--pex-path", @@ -848,23 +842,6 @@ def configure_clp(): help="Add requirements from the given .pex file. This option can be used multiple times.", ) - parser.add_argument( - "--add-dist", - "--add-dists", - dest="distributions", - metavar="FILE", - default=[], - type=str, - action="append", - help=( - "If a wheel, add it to the PEX. If an sdist, build wheels for the selected targets and" - "add them to the PEX. Otherwise, if a directory, add all the distributions found in " - "the given directory to the PEX, building wheels from any sdists first. This option " - "can be used to add a pre-resolved dependency set to a PEX. By default, Pex will " - "ensure the dependencies added form a closure." - ), - ) - register_global_arguments(parser, include_verbosity=True) parser.add_argument( @@ -932,14 +909,6 @@ def _iter_python_sources(python_sources): yield src, dst -def _fingerprint_dist(dist_path): - # type: (str) -> FingerprintedDistribution - return FingerprintedDistribution( - distribution=Distribution.load(dist_path), - fingerprint=CacheHelper.hash(dist_path, hasher=hashlib.sha256), - ) - - def build_pex( requirement_configuration, # type: RequirementConfiguration resolver_configuration, # type: ResolverConfiguration @@ -1045,7 +1014,7 @@ def build_pex( DependencyConfiguration.from_pex_info(requirements_pex_info) ) - if isinstance(resolver_configuration, LockRepositoryConfiguration): + if isinstance(resolver_configuration, (LockRepositoryConfiguration, PreResolvedConfiguration)): pip_configuration = resolver_configuration.pip_configuration elif isinstance(resolver_configuration, PexRepositoryConfiguration): # TODO(John Sirois): Consider finding a way to support custom --index and --find-links in @@ -1060,89 +1029,6 @@ def build_pex( else: pip_configuration = resolver_configuration - with TRACER.timed("Adding local distributions"): - sdists = [] # type: List[str] - wheels = OrderedDict() # type: OrderedDict[str, FingerprintedDistribution] - for dist_or_dir in options.distributions: - abs_dist_or_dir = os.path.expanduser(dist_or_dir) - dists = ( - [abs_dist_or_dir] - if os.path.isfile(abs_dist_or_dir) - else glob.glob(os.path.join(abs_dist_or_dir, "*")) - ) - wheels_to_fingerprint = [] # type: List[str] - for dist in dists: - if not os.path.isfile(dist): - continue - if dist.endswith(".whl"): - wheels_to_fingerprint.append(dist) - elif dist.endswith((".tar.gz", ".tgz", ".tar.bz2", ".tbz2", ".tar.xz", ".txz")): - sdists.append(dist) - if wheels_to_fingerprint: - for fingerprinted_dist in iter_map_parallel( - inputs=wheels_to_fingerprint, - function=_fingerprint_dist, - max_jobs=pip_configuration.max_jobs, - costing_function=lambda whl: os.path.getsize(whl), - noun="wheel", - verb="fingerprint", - verb_past="fingerprinted", - ): - wheels[os.path.basename(fingerprinted_dist.location)] = fingerprinted_dist - if sdists or (wheels and options.pre_install_wheels): - package_index_configuration = PackageIndexConfiguration.create( - pip_version=pip_configuration.version, - resolver_version=pip_configuration.resolver_version, - indexes=pip_configuration.repos_configuration.indexes, - find_links=pip_configuration.repos_configuration.find_links, - network_configuration=pip_configuration.network_configuration, - password_entries=pip_configuration.repos_configuration.password_entries, - use_pip_config=pip_configuration.use_pip_config, - extra_pip_requirements=pip_configuration.extra_requirements, - ) - build_and_install = BuildAndInstallRequest( - build_requests=[ - BuildRequest.create(target=target, source_path=sdist) - for sdist in sdists - for target in targets.unique_targets() - ], - install_requests=[ - InstallRequest( - target=target, wheel_path=wheel.location, fingerprint=wheel.fingerprint - ) - for wheel in wheels.values() - for target in targets.unique_targets() - ], - package_index_configuration=package_index_configuration, - compile=options.compile, - build_configuration=pip_configuration.build_configuration, - verify_wheels=True, - pip_version=pip_configuration.version, - resolver=ConfiguredResolver(pip_configuration=pip_configuration), - dependency_configuration=dependency_config, - ) - resolved_dists = ( - build_and_install.install_distributions( - ignore_errors=options.ignore_errors, - max_parallel_jobs=pip_configuration.max_jobs, - ) - if wheels and options.pre_install_wheels - else build_and_install.build_distributions( - ignore_errors=options.ignore_errors, - max_parallel_jobs=pip_configuration.max_jobs, - ) - ) - dependency_manager.add_from_resolved( - resolved_dist.with_direct_requirements( - [resolved_dist.distribution.as_requirement()] - ) - for resolved_dist in resolved_dists - ) - elif wheels: - for wheel in wheels.values(): - dependency_manager.add_requirement(wheel.distribution.as_requirement()) - dependency_manager.add_distribution(wheel) - project_dependencies = OrderedSet() # type: OrderedSet[Requirement] with TRACER.timed( "Adding distributions built from local projects and collecting their requirements: " @@ -1213,7 +1099,7 @@ def build_pex( for resolved_dist in resolve_result.distributions ), ) - dependency_manager.add_from_resolved(resolve_result.distributions) + dependency_manager.add_from_resolved(resolve_result) dependency_config = resolve_result.dependency_configuration except Unsatisfiable as e: die(str(e)) diff --git a/pex/cli/commands/lock.py b/pex/cli/commands/lock.py index 2e48fa1c2..377454e4a 100644 --- a/pex/cli/commands/lock.py +++ b/pex/cli/commands/lock.py @@ -427,6 +427,7 @@ def _add_resolve_options(cls, parser): cls._create_resolver_options_group(parser), include_pex_repository=False, include_lock=False, + include_pre_resolved=False, ) @classmethod diff --git a/pex/cli/commands/venv.py b/pex/cli/commands/venv.py index 3e8625941..339136f92 100644 --- a/pex/cli/commands/venv.py +++ b/pex/cli/commands/venv.py @@ -110,7 +110,9 @@ def _add_create_arguments(cls, parser): ) installer_options.register(parser) target_options.register(parser, include_platforms=True) - resolver_options.register(parser, include_pex_repository=True, include_lock=True) + resolver_options.register( + parser, include_pex_repository=True, include_lock=True, include_pre_resolved=True + ) requirement_options.register(parser) @classmethod diff --git a/pex/dependency_manager.py b/pex/dependency_manager.py index 351ba0497..c52ea4f30 100644 --- a/pex/dependency_manager.py +++ b/pex/dependency_manager.py @@ -16,11 +16,11 @@ from pex.pep_503 import ProjectName from pex.pex_builder import PEXBuilder from pex.pex_info import PexInfo -from pex.resolve.resolvers import ResolvedDistribution +from pex.resolve.resolvers import ResolveResult from pex.typing import TYPE_CHECKING if TYPE_CHECKING: - from typing import DefaultDict, Iterable + from typing import DefaultDict import attr # vendor:skip else: @@ -59,10 +59,10 @@ def add_from_pex( return pex_info - def add_from_resolved(self, resolved_dists): - # type: (Iterable[ResolvedDistribution]) -> None + def add_from_resolved(self, resolved): + # type: (ResolveResult) -> None - for resolved_dist in resolved_dists: + for resolved_dist in resolved.distributions: for req in resolved_dist.direct_requirements: self.add_requirement(req) self.add_distribution(resolved_dist.fingerprinted_distribution) diff --git a/pex/dist_metadata.py b/pex/dist_metadata.py index d6b3e4636..9140db0de 100644 --- a/pex/dist_metadata.py +++ b/pex/dist_metadata.py @@ -75,14 +75,65 @@ class InvalidMetadataError(MetadataError): """Indicates a metadata value that is invalid.""" +def is_tar_sdist(path): + # type: (Text) -> bool + # N.B.: PEP-625 (https://peps.python.org/pep-0625/) says sdists must use .tar.gz, but we + # have a known example of tar.bz2 in the wild in python-constraint 1.4.0 on PyPI: + # https://pypi.org/project/python-constraint/1.4.0/#files + # This probably all stems from the legacy `python setup.py sdist` as last described here: + # https://docs.python.org/3.11/distutils/sourcedist.html + # There was a move to reject exotic formats in PEP-527 in 2016 and the historical sdist + # formats appear to be listed here: https://peps.python.org/pep-0527/#file-extensions + # A query on the PyPI dataset shows: + # + # SELECT + # REGEXP_EXTRACT(path, r'\.([^.]+|tar\.[^.]+|tar)$') as extension, + # count(*) as count + # FROM `bigquery-public-data.pypi.distribution_metadata` + # group by extension + # order by count desc + # + # | extension | count | + # |-----------|---------| + # | whl | 6332494 | + # * | tar.gz | 5283102 | + # | egg | 135940 | + # * | zip | 108532 | + # | exe | 18452 | + # * | tar.bz2 | 3857 | + # | msi | 625 | + # | rpm | 603 | + # * | tgz | 226 | + # | dmg | 47 | + # | deb | 36 | + # * | tar.zip | 2 | + # * | ZIP | 1 | + return path.lower().endswith((".tar.gz", ".tgz", ".tar.bz2")) + + +def is_zip_sdist(path): + # type: (Text) -> bool + return path.lower().endswith(".zip") + + +def is_sdist(path): + # type: (Text) -> bool + return is_tar_sdist(path) or is_zip_sdist(path) + + +def is_wheel(path): + # type: (Text) -> bool + return path.lower().endswith(".whl") + + def _strip_sdist_path(sdist_path): # type: (Text) -> Optional[Text] - if not sdist_path.endswith((".tar.gz", ".tgz", ".tar.bz2", ".tbz2", ".tar.xz", ".txz", ".zip")): + if not is_sdist(sdist_path): return None sdist_basename = os.path.basename(sdist_path) filename, _ = os.path.splitext(sdist_basename) - if filename.endswith(".tar"): + if filename.lower().endswith(".tar"): filename, _ = os.path.splitext(filename) return filename @@ -336,7 +387,7 @@ def iter_metadata_files( location, MetadataType.DIST_INFO, "*.dist-info", "METADATA" ) ) - elif location.endswith(".whl") and zipfile.is_zipfile(location): + elif is_wheel(location) and zipfile.is_zipfile(location): metadata_files = find_wheel_metadata(location) if metadata_files: listing.append(metadata_files) @@ -347,13 +398,11 @@ def iter_metadata_files( ) ) elif MetadataType.PKG_INFO is metadata_type: - if location.endswith(".zip") and zipfile.is_zipfile(location): + if is_zip_sdist(location) and zipfile.is_zipfile(location): metadata_file = find_zip_sdist_metadata(location) if metadata_file: listing.append(MetadataFiles(metadata=metadata_file)) - elif location.endswith( - (".tar.gz", ".tgz", ".tar.bz2", ".tbz2", ".tar.xz", ".txz") - ) and tarfile.is_tarfile(location): + elif is_tar_sdist(location) and tarfile.is_tarfile(location): metadata_file = find_tar_sdist_metadata(location) if metadata_file: listing.append(MetadataFiles(metadata=metadata_file)) @@ -414,7 +463,7 @@ def from_filename(cls, path): # # The wheel filename convention is specified here: # https://www.python.org/dev/peps/pep-0427/#file-name-convention. - if path.endswith(".whl"): + if is_wheel(path): project_name, version, _ = os.path.basename(path).split("-", 2) return cls(project_name=project_name, version=version) @@ -909,7 +958,7 @@ def of(cls, location): # type: (Text) -> DistributionType.Value if os.path.isdir(location): return cls.INSTALLED - if location.endswith(".whl") and zipfile.is_zipfile(location): + if is_wheel(location) and zipfile.is_zipfile(location): return cls.WHEEL return cls.SDIST diff --git a/pex/environment.py b/pex/environment.py index 2452229fb..1b9317ab4 100644 --- a/pex/environment.py +++ b/pex/environment.py @@ -12,17 +12,18 @@ from pex import dist_metadata, pex_warnings, targets from pex.common import pluralize from pex.dependency_configuration import DependencyConfiguration -from pex.dist_metadata import Distribution, Requirement +from pex.dist_metadata import Distribution, Requirement, is_wheel from pex.fingerprinted_distribution import FingerprintedDistribution from pex.inherit_path import InheritPath from pex.interpreter import PythonInterpreter from pex.layout import ensure_installed, identify_layout from pex.orderedset import OrderedSet -from pex.pep_425 import CompatibilityTags, TagRank +from pex.pep_425 import TagRank from pex.pep_503 import ProjectName from pex.pex_info import PexInfo from pex.targets import Target from pex.third_party.packaging import specifiers +from pex.third_party.packaging.tags import Tag from pex.tracer import TRACER from pex.typing import TYPE_CHECKING @@ -139,7 +140,7 @@ def render_message(self, _target): @attr.s(frozen=True) class _TagMismatch(_UnrankedDistribution): - wheel_tags = attr.ib() # type: CompatibilityTags + wheel_tags = attr.ib() # type: Iterable[Tag] def render_message(self, target): # type: (Target) -> str @@ -332,8 +333,8 @@ def _update_candidate_distributions(self, distribution_iter): def _can_add(self, fingerprinted_dist): # type: (FingerprintedDistribution) -> Union[_RankedDistribution, _UnrankedDistribution] - filename, ext = os.path.splitext(os.path.basename(fingerprinted_dist.location)) - if ext.lower() != ".whl": + filename = os.path.basename(fingerprinted_dist.location) + if not is_wheel(filename): # This supports resolving pex's own vendored distributions which are vendored in a # directory with the project name (`pip/` for pip) and not the corresponding wheel name # (`pip-19.3.1-py2.py3-none-any.whl/` for pip). Pex only vendors universal wheels for @@ -341,23 +342,17 @@ def _can_add(self, fingerprinted_dist): return _RankedDistribution.highest_rank(fingerprinted_dist) try: - wheel_tags = CompatibilityTags.from_wheel(fingerprinted_dist.location) + wheel_eval = self._target.wheel_applies(fingerprinted_dist.distribution) except ValueError: return _InvalidWheelName(fingerprinted_dist, filename) - # There will be multiple parsed tags for compressed tag sets. Ensure we grab the parsed tag - # with highest rank from that expanded set. - best_match = self._target.supported_tags.best_match(wheel_tags) - if best_match is None: - return _TagMismatch(fingerprinted_dist, wheel_tags) + if not wheel_eval.best_match: + return _TagMismatch(fingerprinted_dist, wheel_eval.tags) + if not wheel_eval.applies: + assert wheel_eval.requires_python + return _PythonRequiresMismatch(fingerprinted_dist, wheel_eval.requires_python) - python_requires = dist_metadata.requires_python(fingerprinted_dist.distribution) - if python_requires and not self._target.requires_python_applies( - python_requires, source=fingerprinted_dist.distribution.as_requirement() - ): - return _PythonRequiresMismatch(fingerprinted_dist, python_requires) - - return _RankedDistribution(best_match.rank, fingerprinted_dist) + return _RankedDistribution(wheel_eval.best_match.rank, fingerprinted_dist) def activate(self): # type: () -> Iterable[Distribution] diff --git a/pex/pep_425.py b/pex/pep_425.py index 2d5f35f4b..a32bf9ec3 100644 --- a/pex/pep_425.py +++ b/pex/pep_425.py @@ -6,6 +6,7 @@ import itertools import os.path +from pex.dist_metadata import is_wheel from pex.orderedset import OrderedSet from pex.rank import Rank from pex.third_party.packaging.tags import Tag, parse_tag @@ -56,14 +57,14 @@ class CompatibilityTags(object): @classmethod def from_wheel(cls, wheel): # type: (str) -> CompatibilityTags - wheel_stem, ext = os.path.splitext(os.path.basename(wheel)) - if ".whl" != ext: + if not is_wheel(wheel): raise ValueError( "Can only calculate wheel tags from a filename that ends in .whl per " "https://peps.python.org/pep-0427/#file-name-convention, given: {wheel!r}".format( wheel=wheel ) ) + wheel_stem, _ = os.path.splitext(os.path.basename(wheel)) # Wheel filename format: https://www.python.org/dev/peps/pep-0427/#file-name-convention # `{distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl` wheel_components = wheel_stem.rsplit("-", 3) diff --git a/pex/resolve/configured_resolve.py b/pex/resolve/configured_resolve.py index 76790b7f7..6e95b3260 100644 --- a/pex/resolve/configured_resolve.py +++ b/pex/resolve/configured_resolve.py @@ -3,15 +3,18 @@ from __future__ import absolute_import +from pex.common import pluralize from pex.dependency_configuration import DependencyConfiguration from pex.pep_427 import InstallableType from pex.resolve.configured_resolver import ConfiguredResolver from pex.resolve.lock_resolver import resolve_from_lock from pex.resolve.pex_repository_resolver import resolve_from_pex +from pex.resolve.pre_resolved_resolver import resolve_from_dists from pex.resolve.requirement_configuration import RequirementConfiguration from pex.resolve.resolver_configuration import ( LockRepositoryConfiguration, PexRepositoryConfiguration, + PreResolvedConfiguration, ) from pex.resolve.resolvers import ResolveResult from pex.resolver import resolve as resolve_via_pip @@ -34,6 +37,7 @@ def resolve( dependency_configuration=DependencyConfiguration(), # type: DependencyConfiguration ): # type: (...) -> ResolveResult + if isinstance(resolver_configuration, LockRepositoryConfiguration): lock = try_(resolver_configuration.parse_lock()) with TRACER.timed( @@ -82,6 +86,28 @@ def resolve( result_type=result_type, dependency_configuration=dependency_configuration, ) + elif isinstance(resolver_configuration, PreResolvedConfiguration): + with TRACER.timed( + "Resolving requirements from {sdist_count} pre-resolved {sdists} and " + "{wheel_count} pre-resolved {wheels}.".format( + sdist_count=len(resolver_configuration.sdists), + sdists=pluralize(resolver_configuration.sdists, "sdist"), + wheel_count=len(resolver_configuration.wheels), + wheels=pluralize(resolver_configuration.wheels, "wheel"), + ) + ): + return resolve_from_dists( + targets=targets, + sdists=resolver_configuration.sdists, + wheels=resolver_configuration.wheels, + requirements=requirement_configuration.requirements, + requirement_files=requirement_configuration.requirement_files, + pip_configuration=resolver_configuration.pip_configuration, + compile=compile_pyc, + ignore_errors=ignore_errors, + result_type=result_type, + dependency_configuration=dependency_configuration, + ) else: with TRACER.timed("Resolving requirements."): return resolve_via_pip( diff --git a/pex/resolve/lock_resolver.py b/pex/resolve/lock_resolver.py index 91e2aa28c..475cc06d2 100644 --- a/pex/resolve/lock_resolver.py +++ b/pex/resolve/lock_resolver.py @@ -15,7 +15,7 @@ from pex.common import pluralize from pex.compatibility import cpu_count from pex.dependency_configuration import DependencyConfiguration -from pex.dist_metadata import Requirement +from pex.dist_metadata import Requirement, is_wheel from pex.network_configuration import NetworkConfiguration from pex.orderedset import OrderedSet from pex.pep_427 import InstallableType @@ -414,7 +414,7 @@ def resolve_from_lock( for resolved_subset in subset_result.subsets: for downloadable_artifact in resolved_subset.resolved.downloadable_artifacts: downloaded_artifact = downloaded_artifacts[downloadable_artifact] - if downloaded_artifact.path.endswith(".whl"): + if is_wheel(downloaded_artifact.path): install_requests.append( InstallRequest( target=resolved_subset.target, diff --git a/pex/resolve/locked_resolve.py b/pex/resolve/locked_resolve.py index eb50fd224..347414d23 100644 --- a/pex/resolve/locked_resolve.py +++ b/pex/resolve/locked_resolve.py @@ -10,7 +10,7 @@ from pex.common import pluralize from pex.dependency_configuration import DependencyConfiguration -from pex.dist_metadata import DistMetadata, Requirement +from pex.dist_metadata import DistMetadata, Requirement, is_sdist, is_wheel from pex.enum import Enum from pex.orderedset import OrderedSet from pex.pep_425 import CompatibilityTags, TagRank @@ -161,61 +161,16 @@ def __lt__(self, other): @attr.s(frozen=True, order=False) class FileArtifact(Artifact): - @staticmethod - def is_zip_sdist(path): - # type: (str) -> bool - - # N.B.: Windows sdists traditionally were released in zip format. - return path.endswith(".zip") - - @staticmethod - def is_tar_sdist(path): - # type: (str) -> bool - - # N.B.: PEP-625 (https://peps.python.org/pep-0625/) says sdists must use .tar.gz, but we - # have a known example of tar.bz2 in the wild in python-constraint 1.4.0 on PyPI: - # https://pypi.org/project/python-constraint/1.4.0/#files - # This probably all stems from the legacy `python setup.py sdist` as last described here: - # https://docs.python.org/3.11/distutils/sourcedist.html - # There was a move to reject exotic formats in PEP-527 in 2016 and the historical sdist - # formats appear to be listed here: https://peps.python.org/pep-0527/#file-extensions - # A query on the PyPI dataset shows: - # - # SELECT - # REGEXP_EXTRACT(path, r'\.([^.]+|tar\.[^.]+|tar)$') as extension, - # count(*) as count - # FROM `bigquery-public-data.pypi.distribution_metadata` - # group by extension - # order by count desc - # - # | extension | count | - # |-----------|---------| - # | whl | 6332494 | - # * | tar.gz | 5283102 | - # | egg | 135940 | - # * | zip | 108532 | - # | exe | 18452 | - # * | tar.bz2 | 3857 | - # | msi | 625 | - # | rpm | 603 | - # * | tgz | 226 | - # | dmg | 47 | - # | deb | 36 | - # * | tar.zip | 2 | - # * | ZIP | 1 | - # - return path.endswith((".tar.gz", ".tgz", ".tar.bz2")) - filename = attr.ib() # type: str @property def is_source(self): # type: () -> bool - return self.is_tar_sdist(self.filename) or self.is_zip_sdist(self.filename) + return is_sdist(self.filename) def parse_tags(self): # type: () -> Iterator[tags.Tag] - if self.filename.endswith(".whl"): + if is_wheel(self.filename): for tag in CompatibilityTags.from_wheel(self.filename): yield tag diff --git a/pex/resolve/lockfile/create.py b/pex/resolve/lockfile/create.py index 1ba05f381..1f1ba0dc2 100644 --- a/pex/resolve/lockfile/create.py +++ b/pex/resolve/lockfile/create.py @@ -14,7 +14,7 @@ from pex.build_system import pep_517 from pex.common import open_zip, pluralize, safe_mkdtemp from pex.dependency_configuration import DependencyConfiguration -from pex.dist_metadata import DistMetadata, ProjectNameAndVersion +from pex.dist_metadata import DistMetadata, ProjectNameAndVersion, is_tar_sdist, is_zip_sdist from pex.fetcher import URLFetcher from pex.jobs import Job, Retain, SpawnedJob, execute_parallel from pex.orderedset import OrderedSet @@ -155,10 +155,10 @@ def _prepare_project_directory(build_request): return target, project extract_dir = os.path.join(safe_mkdtemp(), "project") - if FileArtifact.is_zip_sdist(project): + if is_zip_sdist(project): with open_zip(project) as zf: zf.extractall(extract_dir) - elif FileArtifact.is_tar_sdist(project): + elif is_tar_sdist(project): with tarfile.open(project) as tf: tf.extractall(extract_dir) else: diff --git a/pex/resolve/pre_resolved_resolver.py b/pex/resolve/pre_resolved_resolver.py new file mode 100644 index 000000000..9c5af3574 --- /dev/null +++ b/pex/resolve/pre_resolved_resolver.py @@ -0,0 +1,163 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import hashlib +import os +from collections import defaultdict + +from pex.dependency_configuration import DependencyConfiguration +from pex.dist_metadata import Distribution, Requirement +from pex.fingerprinted_distribution import FingerprintedDistribution +from pex.jobs import iter_map_parallel +from pex.pep_427 import InstallableType +from pex.pep_503 import ProjectName +from pex.pip.tool import PackageIndexConfiguration +from pex.requirements import LocalProjectRequirement +from pex.resolve.configured_resolver import ConfiguredResolver +from pex.resolve.requirement_configuration import RequirementConfiguration +from pex.resolve.resolver_configuration import PipConfiguration +from pex.resolve.resolvers import ( + ResolvedDistribution, + ResolveResult, + check_resolve, + sorted_requirements, +) +from pex.resolver import BuildAndInstallRequest, BuildRequest, InstallRequest +from pex.targets import Targets +from pex.typing import TYPE_CHECKING +from pex.util import CacheHelper + +if TYPE_CHECKING: + from typing import DefaultDict, Iterable, List, Optional + + +def _fingerprint_dist(dist_path): + # type: (str) -> FingerprintedDistribution + return FingerprintedDistribution( + distribution=Distribution.load(dist_path), + fingerprint=CacheHelper.hash(dist_path, hasher=hashlib.sha256), + ) + + +def resolve_from_dists( + targets, # type: Targets + sdists, # type: Iterable[str] + wheels, # type: Iterable[str] + requirements=None, # type: Optional[Iterable[str]] + requirement_files=None, # type: Optional[Iterable[str]] + constraint_files=None, # type: Optional[Iterable[str]] + pip_configuration=PipConfiguration(), # type: PipConfiguration + compile=False, # type: bool + ignore_errors=False, # type: bool + result_type=InstallableType.INSTALLED_WHEEL_CHROOT, # type: InstallableType.Value + dependency_configuration=DependencyConfiguration(), # type: DependencyConfiguration +): + # type: (...) -> ResolveResult + + unique_targets = targets.unique_targets() + + requirement_configuration = RequirementConfiguration( + requirements=requirements, + requirement_files=requirement_files, + constraint_files=constraint_files, + ) + + direct_requirements = requirement_configuration.parse_requirements( + pip_configuration.network_configuration + ) + local_projects = [] # type: List[LocalProjectRequirement] + for direct_requirement in direct_requirements: + if isinstance(direct_requirement, LocalProjectRequirement): + local_projects.append(direct_requirement) + + source_paths = [local_project.path for local_project in local_projects] + list( + sdists + ) # type: List[str] + fingerprinted_wheels = tuple( + iter_map_parallel( + inputs=wheels, + function=_fingerprint_dist, + max_jobs=pip_configuration.max_jobs, + costing_function=lambda whl: os.path.getsize(whl), + noun="wheel", + verb="fingerprint", + verb_past="fingerprinted", + ) + ) + + resolved_dists = [] # type: List[ResolvedDistribution] + resolve_installed_wheel_chroots = ( + fingerprinted_wheels and InstallableType.INSTALLED_WHEEL_CHROOT is result_type + ) + if source_paths or resolve_installed_wheel_chroots: + package_index_configuration = PackageIndexConfiguration.create( + pip_version=pip_configuration.version, + resolver_version=pip_configuration.resolver_version, + indexes=pip_configuration.repos_configuration.indexes, + find_links=pip_configuration.repos_configuration.find_links, + network_configuration=pip_configuration.network_configuration, + password_entries=pip_configuration.repos_configuration.password_entries, + use_pip_config=pip_configuration.use_pip_config, + extra_pip_requirements=pip_configuration.extra_requirements, + ) + build_and_install = BuildAndInstallRequest( + build_requests=[ + BuildRequest.create(target=target, source_path=source_path) + for source_path in source_paths + for target in unique_targets + ], + install_requests=[ + InstallRequest( + target=target, wheel_path=wheel.location, fingerprint=wheel.fingerprint + ) + for wheel in fingerprinted_wheels + for target in unique_targets + ], + direct_requirements=direct_requirements, + package_index_configuration=package_index_configuration, + compile=compile, + build_configuration=pip_configuration.build_configuration, + verify_wheels=True, + pip_version=pip_configuration.version, + resolver=ConfiguredResolver(pip_configuration=pip_configuration), + dependency_configuration=dependency_configuration, + ) + resolved_dists.extend( + build_and_install.install_distributions( + ignore_errors=ignore_errors, + max_parallel_jobs=pip_configuration.max_jobs, + ) + if resolve_installed_wheel_chroots + else build_and_install.build_distributions( + ignore_errors=ignore_errors, + max_parallel_jobs=pip_configuration.max_jobs, + ) + ) + elif wheels: + direct_reqs_by_project_name = defaultdict( + list + ) # type: DefaultDict[ProjectName, List[Requirement]] + for parsed_req in direct_requirements: + assert not isinstance(parsed_req, LocalProjectRequirement) + direct_reqs_by_project_name[parsed_req.requirement.project_name].append( + parsed_req.requirement + ) + for wheel in fingerprinted_wheels: + direct_reqs = sorted_requirements(direct_reqs_by_project_name[wheel.project_name]) + for target in unique_targets: + resolved_dists.append( + ResolvedDistribution( + target=target, + fingerprinted_distribution=wheel, + direct_requirements=direct_reqs, + ) + ) + check_resolve(dependency_configuration, resolved_dists) + + return ResolveResult( + dependency_configuration=dependency_configuration, + distributions=tuple(resolved_dists), + type=result_type, + ) diff --git a/pex/resolve/resolved_requirement.py b/pex/resolve/resolved_requirement.py index 5d8546055..7e61da190 100644 --- a/pex/resolve/resolved_requirement.py +++ b/pex/resolve/resolved_requirement.py @@ -7,7 +7,7 @@ from pex import hashing from pex.compatibility import url_unquote, urlparse -from pex.dist_metadata import ProjectNameAndVersion, Requirement +from pex.dist_metadata import ProjectNameAndVersion, Requirement, is_wheel from pex.hashing import HashlibHasher from pex.pep_440 import Version from pex.pep_503 import ProjectName @@ -147,7 +147,7 @@ def parse(cls, url): @property def is_wheel(self): # type: () -> bool - return self.path.endswith(".whl") + return is_wheel(self.path) @property def fingerprint(self): diff --git a/pex/resolve/resolver_configuration.py b/pex/resolve/resolver_configuration.py index 61f788fcd..66f7983d2 100644 --- a/pex/resolve/resolver_configuration.py +++ b/pex/resolve/resolver_configuration.py @@ -220,3 +220,20 @@ def repos_configuration(self): def network_configuration(self): # type: () -> NetworkConfiguration return self.pip_configuration.network_configuration + + +@attr.s(frozen=True) +class PreResolvedConfiguration(object): + sdists = attr.ib() # type: Tuple[str, ...] + wheels = attr.ib() # type: Tuple[str, ...] + pip_configuration = attr.ib() # type: PipConfiguration + + @property + def repos_configuration(self): + # type: () -> ReposConfiguration + return self.pip_configuration.repos_configuration + + @property + def network_configuration(self): + # type: () -> NetworkConfiguration + return self.pip_configuration.network_configuration diff --git a/pex/resolve/resolver_options.py b/pex/resolve/resolver_options.py index 7ff37888d..07d9aa93a 100644 --- a/pex/resolve/resolver_options.py +++ b/pex/resolve/resolver_options.py @@ -3,12 +3,13 @@ from __future__ import absolute_import +import glob import os from argparse import Action, ArgumentTypeError, Namespace, _ActionsContainer from pex import pex_warnings from pex.argparse import HandleBoolAction -from pex.dist_metadata import Requirement +from pex.dist_metadata import Requirement, is_sdist, is_wheel from pex.fetcher import initialize_ssl_context from pex.network_configuration import NetworkConfiguration from pex.orderedset import OrderedSet @@ -23,6 +24,7 @@ LockRepositoryConfiguration, PexRepositoryConfiguration, PipConfiguration, + PreResolvedConfiguration, ReposConfiguration, ResolverVersion, ) @@ -31,7 +33,7 @@ from pex.typing import TYPE_CHECKING, cast if TYPE_CHECKING: - from typing import Optional, Union + from typing import List, Optional, Union class _ManylinuxAction(Action): @@ -64,6 +66,7 @@ def register( parser, # type: _ActionsContainer include_pex_repository=False, # type: bool include_lock=False, # type: bool + include_pre_resolved=False, # type: bool ): # type: (...) -> None """Register resolver configuration options with the given parser. @@ -71,6 +74,7 @@ def register( :param parser: The parser to register resolver configuration options with. :param include_pex_repository: Whether to include the `--pex-repository` option. :param include_lock: Whether to include the `--lock` option. + :param include_pre_resolved: Whether to include the `--add-dist` and `--add-dists` options. """ default_resolver_configuration = PipConfiguration() @@ -151,9 +155,15 @@ def register( help="Deprecated: No longer used.", ) - repository_choice = ( - parser.add_mutually_exclusive_group() if include_pex_repository and include_lock else parser - ) + repository_types = 0 + if include_pex_repository: + repository_types += 1 + if include_lock: + repository_types += 1 + if include_pre_resolved: + repository_types += 1 + + repository_choice = parser.add_mutually_exclusive_group() if repository_types > 1 else parser if include_pex_repository: repository_choice.add_argument( "--pex-repository", @@ -180,6 +190,23 @@ def register( ), ) register_lock_options(parser) + if include_pre_resolved: + repository_choice.add_argument( + "--pre-resolved-dist", + "--pre-resolved-dists", + dest="pre_resolved_dists", + metavar="FILE", + default=[], + type=str, + action="append", + help=( + "If a wheel, add it to the PEX. If an sdist, build wheels for the selected targets " + "and add them to the PEX. Otherwise, if a directory, add all the distributions " + "found in the given directory to the PEX, building wheels from any sdists first. " + "This option can be used to add a pre-resolved dependency set to a PEX. By " + "default, Pex will ensure the dependencies added form a closure." + ), + ) parser.add_argument( "--pre", @@ -481,7 +508,10 @@ class InvalidConfigurationError(Exception): if TYPE_CHECKING: ResolverConfiguration = Union[ - LockRepositoryConfiguration, PexRepositoryConfiguration, PipConfiguration + LockRepositoryConfiguration, + PexRepositoryConfiguration, + PipConfiguration, + PreResolvedConfiguration, ] @@ -494,26 +524,49 @@ def configure(options): """ pex_repository = getattr(options, "pex_repository", None) - lock = getattr(options, "lock", None) - if pex_repository and (options.indexes or options.find_links): - raise InvalidConfigurationError( - 'The "--pex-repository" option cannot be used together with the "--index" or ' - '"--find-links" options.' - ) - if pex_repository: + if options.indexes or options.find_links: + raise InvalidConfigurationError( + 'The "--pex-repository" option cannot be used together with the "--index" or ' + '"--find-links" options.' + ) return PexRepositoryConfiguration( pex_repository=pex_repository, network_configuration=create_network_configuration(options), transitive=options.transitive, ) + pip_configuration = create_pip_configuration(options) + lock = getattr(options, "lock", None) if lock: return LockRepositoryConfiguration( parse_lock=lambda: parse_lockfile(options, lock_file_path=lock), lock_file_path=lock, pip_configuration=pip_configuration, ) + + pre_resolved_dists = getattr(options, "pre_resolved_dists", None) + if pre_resolved_dists: + sdists = [] # type: List[str] + wheels = [] # type: List[str] + for dist_or_dir in pre_resolved_dists: + abs_dist_or_dir = os.path.expanduser(dist_or_dir) + dists = ( + [abs_dist_or_dir] + if os.path.isfile(abs_dist_or_dir) + else glob.glob(os.path.join(abs_dist_or_dir, "*")) + ) + for dist in dists: + if not os.path.isfile(dist): + continue + if is_wheel(dist): + wheels.append(dist) + elif is_sdist(dist): + sdists.append(dist) + return PreResolvedConfiguration( + sdists=tuple(sdists), wheels=tuple(wheels), pip_configuration=pip_configuration + ) + return pip_configuration diff --git a/pex/resolve/resolvers.py b/pex/resolve/resolvers.py index a2e12ebdb..68f679238 100644 --- a/pex/resolve/resolvers.py +++ b/pex/resolve/resolvers.py @@ -3,12 +3,17 @@ from __future__ import absolute_import +import itertools +import os from abc import abstractmethod +from collections import OrderedDict, defaultdict +from pex.common import pluralize from pex.dependency_configuration import DependencyConfiguration from pex.dist_metadata import Distribution, Requirement from pex.fingerprinted_distribution import FingerprintedDistribution from pex.pep_427 import InstallableType +from pex.pep_503 import ProjectName from pex.pip.version import PipVersionValue from pex.resolve.lockfile.model import Lockfile from pex.sorted_tuple import SortedTuple @@ -16,7 +21,7 @@ from pex.typing import TYPE_CHECKING if TYPE_CHECKING: - from typing import Iterable, Optional, Tuple + from typing import DefaultDict, Iterable, List, Optional, Tuple import attr # vendor:skip else: @@ -81,6 +86,90 @@ def with_direct_requirements(self, direct_requirements=None): ) +def check_resolve( + dependency_configuration, # type: DependencyConfiguration + resolved_distributions, # type: Iterable[ResolvedDistribution] +): + # type: (...) -> None + resolved_distributions_by_project_name = ( + OrderedDict() + ) # type: OrderedDict[ProjectName, List[ResolvedDistribution]] + for resolved_distribution in resolved_distributions: + resolved_distributions_by_project_name.setdefault( + resolved_distribution.distribution.metadata.project_name, [] + ).append(resolved_distribution) + + unsatisfied = defaultdict(list) # type: DefaultDict[Target, List[str]] + for resolved_distribution in itertools.chain.from_iterable( + resolved_distributions_by_project_name.values() + ): + dist = resolved_distribution.distribution + target = resolved_distribution.target + + for requirement in dist.requires(): + if dependency_configuration.excluded_by(requirement): + continue + requirement = ( + dependency_configuration.overridden_by(requirement, target=target) or requirement + ) + if not target.requirement_applies(requirement): + continue + + installed_requirement_dists = resolved_distributions_by_project_name.get( + requirement.project_name + ) + if not installed_requirement_dists: + unsatisfied[target].append( + "{dist} requires {requirement} but no version was resolved".format( + dist=dist.as_requirement(), requirement=requirement + ) + ) + else: + resolved_dists = [ + installed_requirement_dist.distribution + for installed_requirement_dist in installed_requirement_dists + ] + if not any( + ( + requirement.specifier.contains(resolved_dist.version, prereleases=True) + and target.wheel_applies(resolved_dist) + ) + for resolved_dist in resolved_dists + ): + unsatisfied[target].append( + "{dist} requires {requirement} but {count} incompatible {dists_were} " + "resolved:\n {dists}".format( + dist=dist, + requirement=requirement, + count=len(resolved_dists), + dists_were="dists were" if len(resolved_dists) > 1 else "dist was", + dists="\n ".join( + os.path.basename(resolved_dist.location) + for resolved_dist in resolved_dists + ), + ) + ) + + if unsatisfied: + unsatisfieds = [] + for target, missing in unsatisfied.items(): + unsatisfieds.append( + "{target} is not compatible with:\n {missing}".format( + target=target.render_description(), missing="\n ".join(missing) + ) + ) + raise Unsatisfiable( + "Failed to resolve compatible distributions for {count} {targets}:\n{failures}".format( + count=len(unsatisfieds), + targets=pluralize(unsatisfieds, "target"), + failures="\n".join( + "{index}: {failure}".format(index=index, failure=failure) + for index, failure in enumerate(unsatisfieds, start=1) + ), + ) + ) + + @attr.s(frozen=True) class ResolveResult(object): dependency_configuration = attr.ib() # type: DependencyConfiguration diff --git a/pex/resolver.py b/pex/resolver.py index 263d24370..47e495bd2 100644 --- a/pex/resolver.py +++ b/pex/resolver.py @@ -20,13 +20,12 @@ from pex.common import pluralize, safe_mkdir, safe_mkdtemp from pex.compatibility import url_unquote, urlparse from pex.dependency_configuration import DependencyConfiguration -from pex.dist_metadata import DistMetadata, Distribution, ProjectNameAndVersion, Requirement +from pex.dist_metadata import DistMetadata, Distribution, Requirement, is_wheel from pex.fingerprinted_distribution import FingerprintedDistribution from pex.jobs import Raise, SpawnedJob, execute_parallel, iter_map_parallel from pex.network_configuration import NetworkConfiguration from pex.orderedset import OrderedSet from pex.pep_376 import InstalledWheel -from pex.pep_425 import CompatibilityTags from pex.pep_427 import InstallableType, WheelError, install_wheel_chroot from pex.pep_503 import ProjectName from pex.pip.download_observer import DownloadObserver @@ -43,6 +42,7 @@ ResolveResult, Unsatisfiable, Untranslatable, + check_resolve, ) from pex.targets import LocalInterpreter, Target, Targets from pex.tracer import TRACER @@ -170,7 +170,7 @@ class DownloadResult(object): @staticmethod def _is_wheel(path): # type: (str) -> bool - return os.path.isfile(path) and path.endswith(".whl") + return is_wheel(path) and zipfile.is_zipfile(path) target = attr.ib() # type: Target download_dir = attr.ib() # type: str @@ -318,9 +318,8 @@ def finalize_build(self, check_compatible=True): ) wheel_path = wheels[0] if check_compatible and self.request.target.is_foreign: - wheel_tags = CompatibilityTags.from_wheel(wheel_path) - if not self.request.target.supported_tags.compatible_tags(wheel_tags): - project_name_and_version = ProjectNameAndVersion.from_filename(wheel_path) + wheel = Distribution.load(wheel_path) + if not self.request.target.wheel_applies(wheel): raise ValueError( "No pre-built wheel was available for {project_name} {version}.{eol}" "Successfully built the wheel {wheel} from the sdist {sdist} but it is not " @@ -328,8 +327,8 @@ def finalize_build(self, check_compatible=True): "You'll need to build a wheel from {sdist} on the foreign target platform and " "make it available to Pex via a `--find-links` repo or a custom " "`--index`.".format( - project_name=project_name_and_version.project_name, - version=project_name_and_version.version, + project_name=wheel.project_name, + version=wheel.version, eol=os.linesep, wheel=os.path.basename(wheel_path), sdist=os.path.basename(self.request.source_path), @@ -789,7 +788,7 @@ def _resolve_direct_file_deps( "The {wheel} wheel has a dependency on {url} which does not exist on this " "machine.".format(wheel=install_request.wheel_file, url=requirement.url) ) - if dist_path.endswith(".whl"): + if is_wheel(dist_path): to_install.add(InstallRequest.create(install_request.target, dist_path)) else: to_build.add(BuildRequest.create(install_request.target, dist_path)) @@ -873,7 +872,7 @@ def build_distributions( if not ignore_errors: with TRACER.timed("Checking build"): - self._check(wheels) + check_resolve(self._dependency_configuration, wheels) return direct_requirements.adjust(wheels) def install_distributions( @@ -946,77 +945,9 @@ def add_installation(install_result): if not ignore_errors: with TRACER.timed("Checking install"): - self._check(installations) + check_resolve(self._dependency_configuration, installations) return direct_requirements.adjust(installations) - def _check(self, resolved_distributions): - # type: (Iterable[ResolvedDistribution]) -> None - resolved_distributions_by_project_name = ( - OrderedDict() - ) # type: OrderedDict[ProjectName, List[ResolvedDistribution]] - for resolved_distribution in resolved_distributions: - resolved_distributions_by_project_name.setdefault( - resolved_distribution.distribution.metadata.project_name, [] - ).append(resolved_distribution) - - unsatisfied = [] - for resolved_distribution in itertools.chain.from_iterable( - resolved_distributions_by_project_name.values() - ): - dist = resolved_distribution.distribution - target = resolved_distribution.target - for requirement in dist.requires(): - if self._dependency_configuration.excluded_by(requirement): - continue - requirement = ( - self._dependency_configuration.overridden_by(requirement, target=target) - or requirement - ) - if not target.requirement_applies(requirement): - continue - - installed_requirement_dists = resolved_distributions_by_project_name.get( - requirement.project_name - ) - if not installed_requirement_dists: - unsatisfied.append( - "{dist} requires {requirement} but no version was resolved".format( - dist=dist.as_requirement(), requirement=requirement - ) - ) - else: - resolved_dists = [ - installed_requirement_dist.distribution - for installed_requirement_dist in installed_requirement_dists - ] - if not any( - requirement.specifier.contains(resolved_dist.version, prereleases=True) - for resolved_dist in resolved_dists - ): - unsatisfied.append( - "{dist} requires {requirement} but {count} incompatible {dists_were} " - "resolved: {dists}".format( - dist=dist.as_requirement(), - requirement=requirement, - count=len(resolved_dists), - dists_were="dists were" if len(resolved_dists) > 1 else "dist was", - dists=" ".join( - os.path.basename(resolved_dist.location) - for resolved_dist in resolved_dists - ), - ) - ) - - if unsatisfied: - raise Unsatisfiable( - "Failed to resolve compatible distributions:\n{failures}".format( - failures="\n".join( - "{index}: {failure}".format(index=index, failure=failure) - for index, failure in enumerate(unsatisfied, start=1) - ) - ) - ) - def _parse_reqs( requirements=None, # type: Optional[Iterable[str]] @@ -1265,7 +1196,7 @@ def _calculate_fingerprint(self): @property def is_wheel(self): - return self.path.endswith(".whl") and zipfile.is_zipfile(self.path) + return is_wheel(self.path) and zipfile.is_zipfile(self.path) @attr.s(frozen=True) diff --git a/pex/targets.py b/pex/targets.py index 37c02f229..49ad956f5 100644 --- a/pex/targets.py +++ b/pex/targets.py @@ -5,14 +5,15 @@ import os -from pex.dist_metadata import Requirement +from pex.dist_metadata import Distribution, Requirement from pex.interpreter import PythonInterpreter, calculate_binary_name from pex.orderedset import OrderedSet -from pex.pep_425 import CompatibilityTags +from pex.pep_425 import CompatibilityTags, RankedTag from pex.pep_508 import MarkerEnvironment from pex.platforms import Platform from pex.result import Error from pex.third_party.packaging.specifiers import SpecifierSet +from pex.third_party.packaging.tags import Tag from pex.typing import TYPE_CHECKING, cast if TYPE_CHECKING: @@ -27,6 +28,21 @@ class RequiresPythonError(Exception): """Indicates the impossibility of evaluating Requires-Python metadata.""" +@attr.s(frozen=True) +class WheelEvaluation(object): + tags = attr.ib() # type: Tuple[Tag, ...] + best_match = attr.ib() # type: Optional[RankedTag] + requires_python = attr.ib() # type: Optional[SpecifierSet] + applies = attr.ib() # type: bool + + def __bool__(self): + # type: () -> bool + return self.applies + + # N.B.: For Python 2.7. + __nonzero__ = __bool__ + + @attr.s(frozen=True) class Target(object): id = attr.ib() # type: str @@ -153,6 +169,25 @@ def requirement_applies( return False + def wheel_applies(self, wheel): + # type: (Distribution) -> WheelEvaluation + wheel_tags = CompatibilityTags.from_wheel(wheel.location) + ranked_tag = self.supported_tags.best_match(wheel_tags) + return WheelEvaluation( + tags=tuple(wheel_tags), + best_match=ranked_tag, + requires_python=wheel.metadata.requires_python, + applies=( + ranked_tag is not None + and ( + not wheel.metadata.requires_python + or self.requires_python_applies( + wheel.metadata.requires_python, source=wheel.location + ) + ) + ), + ) + def __str__(self): # type: () -> str return str(self.platform.tag) diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index 5a3367531..3769243e8 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -20,7 +20,7 @@ from pex.cache.dirs import CacheDir from pex.common import is_exe, safe_mkdir, safe_open, safe_rmtree, temporary_dir, touch from pex.compatibility import WINDOWS, commonpath -from pex.dist_metadata import Distribution, Requirement +from pex.dist_metadata import Distribution, Requirement, is_wheel from pex.fetcher import URLFetcher from pex.interpreter import PythonInterpreter from pex.layout import Layout @@ -1360,7 +1360,7 @@ def iter_distributions(pex_root, project_name): for d in dirs: if not d.startswith(project_name): continue - if not d.endswith(".whl"): + if not is_wheel(d): continue wheel_path = os.path.realpath(os.path.join(root, d)) if wheel_path in found: @@ -1843,7 +1843,7 @@ def test_constraint_file_from_url(tmpdir): assert len(dist_paths) == 3 dist_paths.remove("fasteners-0.15-py2.py3-none-any.whl") for dist_path in dist_paths: - assert dist_path.startswith(("six-", "monotonic-")) and dist_path.endswith(".whl") + assert dist_path.startswith(("six-", "monotonic-")) and is_wheel(dist_path) def test_console_script_from_pex_path(tmpdir): diff --git a/tests/integration/test_issue_1179.py b/tests/integration/test_issue_1179.py index 3e68b1f92..66488f71c 100644 --- a/tests/integration/test_issue_1179.py +++ b/tests/integration/test_issue_1179.py @@ -2,9 +2,11 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). import sys +from textwrap import dedent import pytest +from pex.targets import LocalInterpreter from testing import run_pex_command @@ -27,10 +29,17 @@ def test_pip_2020_resolver_engaged(): args=["--resolver-version", "pip-legacy-resolver"] + pex_args, quiet=True ) results.assert_failure() - assert "Failed to resolve compatible distributions:" in results.error assert ( - "1: boto3==1.15.6 requires botocore<1.19.0,>=1.18.6 but 1 incompatible dist was resolved: " - "botocore-1.19.63-py2.py3-none-any.whl" in results.error + dedent( + """\ + Failed to resolve compatible distributions for 1 target: + 1: {target} is not compatible with: + boto3 1.15.6 requires botocore<1.19.0,>=1.18.6 but 1 incompatible dist was resolved: + botocore-1.19.63-py2.py3-none-any.whl + """.format( + target=LocalInterpreter.create().render_description() + ) + ) + in results.error ), results.error - run_pex_command(args=["--resolver-version", "pip-2020-resolver"] + pex_args).assert_success() diff --git a/tests/test_dist_metadata.py b/tests/test_dist_metadata.py index 4297e8150..bfc1f914c 100644 --- a/tests/test_dist_metadata.py +++ b/tests/test_dist_metadata.py @@ -69,7 +69,7 @@ def downloaded_sdist(requirement): dists = os.listdir(download_dir) assert len(dists) == 1, "Expected 1 dist to be downloaded for {}.".format(requirement) sdist = os.path.join(download_dir, dists[0]) - assert sdist.endswith((".sdist", ".tar.gz", ".zip")) + assert sdist.endswith((".tar.gz", ".zip")) yield sdist diff --git a/tests/test_resolver.py b/tests/test_resolver.py index b068a75b2..8af21d84f 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -12,7 +12,7 @@ import pytest -from pex import targets +from pex import dist_metadata, targets from pex.build_system.pep_517 import build_sdist from pex.common import safe_copy, safe_mkdtemp, temporary_dir from pex.dist_metadata import Distribution, Requirement @@ -523,7 +523,9 @@ def assert_dist( dist = distributions_by_name[project_name] assert version == dist.version - assert is_wheel == (dist.location.endswith(".whl") and zipfile.is_zipfile(dist.location)) + assert is_wheel == ( + dist_metadata.is_wheel(dist.location) and zipfile.is_zipfile(dist.location) + ) assert_dist("project1", "1.0.0", is_wheel=False) assert_dist("project2", "2.0.0", is_wheel=True)