Skip to content

Commit

Permalink
Refactor to a PreResolvedResolver.
Browse files Browse the repository at this point in the history 
Also fix up resolve checking for both the Pip resolver and the
pre-resolver.
  • Loading branch information
@jsirois
jsirois committed Sep 12, 2024
1 parent 7681b1c commit d638a95
Show file tree
Hide file tree
Showing 22 changed files with 531 additions and 317 deletions.
128 changes: 7 additions & 121 deletions pex/bin/pex.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,12 @@

from __future__ import absolute_import, print_function

import glob
import hashlib
import itertools
import json
import os
import shlex
import sys
from argparse import Action, ArgumentDefaultsHelpFormatter, ArgumentError, ArgumentParser
from collections import OrderedDict
from textwrap import TextWrapper

from pex import dependency_configuration, pex_warnings, repl, scie
Expand All @@ -29,14 +26,12 @@
from pex.common import CopyMode, die, is_pyc_dir, is_pyc_file, safe_mkdtemp
from pex.dependency_configuration import DependencyConfiguration
from pex.dependency_manager import DependencyManager
from pex.dist_metadata import Distribution, Requirement
from pex.dist_metadata import Requirement
from pex.docs.command import serve_html_docs
from pex.enum import Enum
from pex.fetcher import URLFetcher
from pex.fingerprinted_distribution import FingerprintedDistribution
from pex.inherit_path import InheritPath
from pex.interpreter_constraints import InterpreterConstraint, InterpreterConstraints
from pex.jobs import iter_map_parallel
from pex.layout import Layout, ensure_installed
from pex.orderedset import OrderedSet
from pex.pep_427 import InstallableType
Expand All @@ -45,7 +40,6 @@
from pex.pex_bootstrapper import ensure_venv
from pex.pex_builder import Check, PEXBuilder
from pex.pex_info import PexInfo
from pex.pip.tool import PackageIndexConfiguration
from pex.resolve import (
project,
requirement_options,
Expand All @@ -55,21 +49,19 @@
)
from pex.resolve.config import finalize as finalize_resolve_config
from pex.resolve.configured_resolve import resolve
from pex.resolve.configured_resolver import ConfiguredResolver
from pex.resolve.requirement_configuration import RequirementConfiguration
from pex.resolve.resolver_configuration import (
LockRepositoryConfiguration,
PexRepositoryConfiguration,
PreResolvedConfiguration,
)
from pex.resolve.resolver_options import create_pip_configuration
from pex.resolve.resolvers import Unsatisfiable, sorted_requirements
from pex.resolver import BuildAndInstallRequest, BuildRequest, InstallRequest
from pex.result import Error, ResultError, catch, try_
from pex.scie import ScieConfiguration
from pex.targets import Targets
from pex.tracer import TRACER
from pex.typing import TYPE_CHECKING, cast
from pex.util import CacheHelper
from pex.variables import ENV, Variables
from pex.venv.bin_path import BinPath
from pex.version import __version__
Expand Down Expand Up @@ -145,7 +137,9 @@ def configure_clp_pex_resolution(parser):
),
)

resolver_options.register(group, include_pex_repository=True, include_lock=True)
resolver_options.register(
group, include_pex_repository=True, include_lock=True, include_pre_resolved=True
)

group.add_argument(
"--pex-path",
Expand Down Expand Up @@ -848,23 +842,6 @@ def configure_clp():
help="Add requirements from the given .pex file. This option can be used multiple times.",
)

parser.add_argument(
"--add-dist",
"--add-dists",
dest="distributions",
metavar="FILE",
default=[],
type=str,
action="append",
help=(
"If a wheel, add it to the PEX. If an sdist, build wheels for the selected targets and"
"add them to the PEX. Otherwise, if a directory, add all the distributions found in "
"the given directory to the PEX, building wheels from any sdists first. This option "
"can be used to add a pre-resolved dependency set to a PEX. By default, Pex will "
"ensure the dependencies added form a closure."
),
)

register_global_arguments(parser, include_verbosity=True)

parser.add_argument(
Expand Down Expand Up @@ -932,14 +909,6 @@ def _iter_python_sources(python_sources):
yield src, dst


def _fingerprint_dist(dist_path):
# type: (str) -> FingerprintedDistribution
return FingerprintedDistribution(
distribution=Distribution.load(dist_path),
fingerprint=CacheHelper.hash(dist_path, hasher=hashlib.sha256),
)


def build_pex(
requirement_configuration, # type: RequirementConfiguration
resolver_configuration, # type: ResolverConfiguration
Expand Down Expand Up @@ -1045,7 +1014,7 @@ def build_pex(
DependencyConfiguration.from_pex_info(requirements_pex_info)
)

if isinstance(resolver_configuration, LockRepositoryConfiguration):
if isinstance(resolver_configuration, (LockRepositoryConfiguration, PreResolvedConfiguration)):
pip_configuration = resolver_configuration.pip_configuration
elif isinstance(resolver_configuration, PexRepositoryConfiguration):
# TODO(John Sirois): Consider finding a way to support custom --index and --find-links in
Expand All @@ -1060,89 +1029,6 @@ def build_pex(
else:
pip_configuration = resolver_configuration

with TRACER.timed("Adding local distributions"):
sdists = [] # type: List[str]
wheels = OrderedDict() # type: OrderedDict[str, FingerprintedDistribution]
for dist_or_dir in options.distributions:
abs_dist_or_dir = os.path.expanduser(dist_or_dir)
dists = (
[abs_dist_or_dir]
if os.path.isfile(abs_dist_or_dir)
else glob.glob(os.path.join(abs_dist_or_dir, "*"))
)
wheels_to_fingerprint = [] # type: List[str]
for dist in dists:
if not os.path.isfile(dist):
continue
if dist.endswith(".whl"):
wheels_to_fingerprint.append(dist)
elif dist.endswith((".tar.gz", ".tgz", ".tar.bz2", ".tbz2", ".tar.xz", ".txz")):
sdists.append(dist)
if wheels_to_fingerprint:
for fingerprinted_dist in iter_map_parallel(
inputs=wheels_to_fingerprint,
function=_fingerprint_dist,
max_jobs=pip_configuration.max_jobs,
costing_function=lambda whl: os.path.getsize(whl),
noun="wheel",
verb="fingerprint",
verb_past="fingerprinted",
):
wheels[os.path.basename(fingerprinted_dist.location)] = fingerprinted_dist
if sdists or (wheels and options.pre_install_wheels):
package_index_configuration = PackageIndexConfiguration.create(
pip_version=pip_configuration.version,
resolver_version=pip_configuration.resolver_version,
indexes=pip_configuration.repos_configuration.indexes,
find_links=pip_configuration.repos_configuration.find_links,
network_configuration=pip_configuration.network_configuration,
password_entries=pip_configuration.repos_configuration.password_entries,
use_pip_config=pip_configuration.use_pip_config,
extra_pip_requirements=pip_configuration.extra_requirements,
)
build_and_install = BuildAndInstallRequest(
build_requests=[
BuildRequest.create(target=target, source_path=sdist)
for sdist in sdists
for target in targets.unique_targets()
],
install_requests=[
InstallRequest(
target=target, wheel_path=wheel.location, fingerprint=wheel.fingerprint
)
for wheel in wheels.values()
for target in targets.unique_targets()
],
package_index_configuration=package_index_configuration,
compile=options.compile,
build_configuration=pip_configuration.build_configuration,
verify_wheels=True,
pip_version=pip_configuration.version,
resolver=ConfiguredResolver(pip_configuration=pip_configuration),
dependency_configuration=dependency_config,
)
resolved_dists = (
build_and_install.install_distributions(
ignore_errors=options.ignore_errors,
max_parallel_jobs=pip_configuration.max_jobs,
)
if wheels and options.pre_install_wheels
else build_and_install.build_distributions(
ignore_errors=options.ignore_errors,
max_parallel_jobs=pip_configuration.max_jobs,
)
)
dependency_manager.add_from_resolved(
resolved_dist.with_direct_requirements(
[resolved_dist.distribution.as_requirement()]
)
for resolved_dist in resolved_dists
)
elif wheels:
for wheel in wheels.values():
dependency_manager.add_requirement(wheel.distribution.as_requirement())
dependency_manager.add_distribution(wheel)

project_dependencies = OrderedSet() # type: OrderedSet[Requirement]
with TRACER.timed(
"Adding distributions built from local projects and collecting their requirements: "
Expand Down Expand Up @@ -1213,7 +1099,7 @@ def build_pex(
for resolved_dist in resolve_result.distributions
),
)
dependency_manager.add_from_resolved(resolve_result.distributions)
dependency_manager.add_from_resolved(resolve_result)
dependency_config = resolve_result.dependency_configuration
except Unsatisfiable as e:
die(str(e))
Expand Down
1 change: 1 addition & 0 deletions pex/cli/commands/lock.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -427,6 +427,7 @@ def _add_resolve_options(cls, parser):
cls._create_resolver_options_group(parser),
include_pex_repository=False,
include_lock=False,
include_pre_resolved=False,
)

@classmethod
Expand Down
4 changes: 3 additions & 1 deletion pex/cli/commands/venv.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,9 @@ def _add_create_arguments(cls, parser):
)
installer_options.register(parser)
target_options.register(parser, include_platforms=True)
resolver_options.register(parser, include_pex_repository=True, include_lock=True)
resolver_options.register(
parser, include_pex_repository=True, include_lock=True, include_pre_resolved=True
)
requirement_options.register(parser)

@classmethod
Expand Down
10 changes: 5 additions & 5 deletions pex/dependency_manager.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,11 @@
from pex.pep_503 import ProjectName
from pex.pex_builder import PEXBuilder
from pex.pex_info import PexInfo
from pex.resolve.resolvers import ResolvedDistribution
from pex.resolve.resolvers import ResolveResult
from pex.typing import TYPE_CHECKING

if TYPE_CHECKING:
from typing import DefaultDict, Iterable
from typing import DefaultDict

import attr # vendor:skip
else:
Expand Down Expand Up @@ -59,10 +59,10 @@ def add_from_pex(

return pex_info

def add_from_resolved(self, resolved_dists):
# type: (Iterable[ResolvedDistribution]) -> None
def add_from_resolved(self, resolved):
# type: (ResolveResult) -> None

for resolved_dist in resolved_dists:
for resolved_dist in resolved.distributions:
for req in resolved_dist.direct_requirements:
self.add_requirement(req)
self.add_distribution(resolved_dist.fingerprinted_distribution)
Expand Down
67 changes: 58 additions & 9 deletions pex/dist_metadata.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,14 +75,65 @@ class InvalidMetadataError(MetadataError):
"""Indicates a metadata value that is invalid."""


def is_tar_sdist(path):
# type: (Text) -> bool
# N.B.: PEP-625 (https://peps.python.org/pep-0625/) says sdists must use .tar.gz, but we
# have a known example of tar.bz2 in the wild in python-constraint 1.4.0 on PyPI:
# https://pypi.org/project/python-constraint/1.4.0/#files
# This probably all stems from the legacy `python setup.py sdist` as last described here:
# https://docs.python.org/3.11/distutils/sourcedist.html
# There was a move to reject exotic formats in PEP-527 in 2016 and the historical sdist
# formats appear to be listed here: https://peps.python.org/pep-0527/#file-extensions
# A query on the PyPI dataset shows:
#
# SELECT
# REGEXP_EXTRACT(path, r'\.([^.]+|tar\.[^.]+|tar)$') as extension,
# count(*) as count
# FROM `bigquery-public-data.pypi.distribution_metadata`
# group by extension
# order by count desc
#
# | extension | count |
# |-----------|---------|
# | whl | 6332494 |
# * | tar.gz | 5283102 |
# | egg | 135940 |
# * | zip | 108532 |
# | exe | 18452 |
# * | tar.bz2 | 3857 |
# | msi | 625 |
# | rpm | 603 |
# * | tgz | 226 |
# | dmg | 47 |
# | deb | 36 |
# * | tar.zip | 2 |
# * | ZIP | 1 |
return path.lower().endswith((".tar.gz", ".tgz", ".tar.bz2"))


def is_zip_sdist(path):
# type: (Text) -> bool
return path.lower().endswith(".zip")


def is_sdist(path):
# type: (Text) -> bool
return is_tar_sdist(path) or is_zip_sdist(path)


def is_wheel(path):
# type: (Text) -> bool
return path.lower().endswith(".whl")


def _strip_sdist_path(sdist_path):
# type: (Text) -> Optional[Text]
if not sdist_path.endswith((".tar.gz", ".tgz", ".tar.bz2", ".tbz2", ".tar.xz", ".txz", ".zip")):
if not is_sdist(sdist_path):
return None

sdist_basename = os.path.basename(sdist_path)
filename, _ = os.path.splitext(sdist_basename)
if filename.endswith(".tar"):
if filename.lower().endswith(".tar"):
filename, _ = os.path.splitext(filename)
return filename

Expand Down Expand Up @@ -336,7 +387,7 @@ def iter_metadata_files(
location, MetadataType.DIST_INFO, "*.dist-info", "METADATA"
)
)
elif location.endswith(".whl") and zipfile.is_zipfile(location):
elif is_wheel(location) and zipfile.is_zipfile(location):
metadata_files = find_wheel_metadata(location)
if metadata_files:
listing.append(metadata_files)
Expand All @@ -347,13 +398,11 @@ def iter_metadata_files(
)
)
elif MetadataType.PKG_INFO is metadata_type:
if location.endswith(".zip") and zipfile.is_zipfile(location):
if is_zip_sdist(location) and zipfile.is_zipfile(location):
metadata_file = find_zip_sdist_metadata(location)
if metadata_file:
listing.append(MetadataFiles(metadata=metadata_file))
elif location.endswith(
(".tar.gz", ".tgz", ".tar.bz2", ".tbz2", ".tar.xz", ".txz")
) and tarfile.is_tarfile(location):
elif is_tar_sdist(location) and tarfile.is_tarfile(location):
metadata_file = find_tar_sdist_metadata(location)
if metadata_file:
listing.append(MetadataFiles(metadata=metadata_file))
Expand Down Expand Up @@ -414,7 +463,7 @@ def from_filename(cls, path):
#
# The wheel filename convention is specified here:
# https://www.python.org/dev/peps/pep-0427/#file-name-convention.
if path.endswith(".whl"):
if is_wheel(path):
project_name, version, _ = os.path.basename(path).split("-", 2)
return cls(project_name=project_name, version=version)

Expand Down Expand Up @@ -909,7 +958,7 @@ def of(cls, location):
# type: (Text) -> DistributionType.Value
if os.path.isdir(location):
return cls.INSTALLED
if location.endswith(".whl") and zipfile.is_zipfile(location):
if is_wheel(location) and zipfile.is_zipfile(location):
return cls.WHEEL
return cls.SDIST

Expand Down
Loading

0 comments on commit d638a95

Please sign in to comment.