Re-work Locker state-machine.

Avoid parsing name and version from anything but wheels, where there is
a hard standard.

pex/pip/download_observer.py

@@ -4,7 +4,7 @@
 from __future__ import absolute_import
 
 from pex.pip.log_analyzer import LogAnalyzer
-from pex.typing import TYPE_CHECKING, Generic
+from pex.typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from typing import Iterable, Mapping, Optional, Text
@@ -21,28 +21,7 @@ class Patch(object):
     env = attr.ib(factory=dict)  # type: Mapping[str, str]
 
 
-if TYPE_CHECKING:
-    from typing import TypeVar
-
-    _L = TypeVar("_L", bound=LogAnalyzer)
-
-
-class DownloadObserver(Generic["_L"]):
-    def __init__(
-        self,
-        analyzer,  # type: _L
-        patch=Patch(),  # type: Patch
-    ):
-        # type: (...) -> None
-        self._analyzer = analyzer
-        self._patch = patch
-
-    @property
-    def analyzer(self):
-        # type: () -> _L
-        return self._analyzer
-
-    @property
-    def patch(self):
-        # type: () -> Patch
-        return self._patch
+@attr.s(frozen=True)
+class DownloadObserver(object):
+    analyzer = attr.ib()  # type: Optional[LogAnalyzer]
+    patch = attr.ib()  # type: Patch

pex/pip/tool.py

@@ -475,7 +475,8 @@ def spawn_download_distributions(
         code = None  # type: Optional[Text]
         for obs in (foreign_platform_observer, observer):
             if obs:
-                log_analyzers.append(obs.analyzer)
+                if obs.analyzer:
+                    log_analyzers.append(obs.analyzer)
                 download_cmd.extend(obs.patch.args)
                 extra_env.update(obs.patch.env)
                 code = code or obs.patch.code

pex/pip/vcs.py

@@ -3,12 +3,12 @@
 
 from __future__ import absolute_import
 
-import hashlib
 import os
 import re
 
 from pex import hashing
 from pex.common import filter_pyc_dirs, filter_pyc_files, open_zip, temporary_dir
+from pex.hashing import Sha256
 from pex.pep_440 import Version
 from pex.pep_503 import ProjectName
 from pex.requirements import VCS
@@ -18,7 +18,7 @@
 from pex.typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from typing import Union
+    from typing import Tuple, Union
 
     from pex.hashing import HintedDigest
 
@@ -61,16 +61,16 @@ def fingerprint_downloaded_vcs_archive(
     version,  # type: str
     vcs,  # type: VCS.Value
 ):
-    # type: (...) -> Fingerprint
+    # type: (...) -> Tuple[Fingerprint, str]
 
     archive_path = try_(
         _find_built_source_dist(
             build_dir=download_dir, project_name=ProjectName(project_name), version=Version(version)
         )
     )
-    digest = hashlib.sha256()
+    digest = Sha256()
     digest_vcs_archive(archive_path=archive_path, vcs=vcs, digest=digest)
-    return Fingerprint(algorithm=digest.name, hash=digest.hexdigest())
+    return Fingerprint.from_digest(digest), archive_path
 
 
 def digest_vcs_archive(

pex/resolve/downloads.py

@@ -7,16 +7,13 @@
 from pex.atomic_directory import atomic_directory
 from pex.common import safe_mkdir, safe_mkdtemp
 from pex.compatibility import unquote, urlparse
-from pex.fetcher import URLFetcher
 from pex.hashing import Sha256
 from pex.jobs import Job, Raise, SpawnedJob, execute_parallel
-from pex.pep_503 import ProjectName
+from pex.pip.download_observer import DownloadObserver
 from pex.pip.installation import get_pip
 from pex.pip.tool import PackageIndexConfiguration, Pip
-from pex.requirements import parse_requirement_string
 from pex.resolve import locker
 from pex.resolve.locked_resolve import Artifact, FileArtifact, LockConfiguration, LockStyle
-from pex.resolve.pep_691.fingerprint_service import FingerprintService
 from pex.resolve.resolved_requirement import Fingerprint, PartialArtifact
 from pex.resolve.resolvers import Resolver
 from pex.result import Error
@@ -30,7 +27,6 @@
     import attr  # vendor:skip
 
     from pex.hashing import HintedDigest
-    from pex.requirements import ParsedRequirement
 else:
     from pex.third_party import attr
 
@@ -77,7 +73,7 @@ def _fingerprint_and_move(path):
         with atomic_directory(target_dir) as atomic_dir:
             if not atomic_dir.is_finalized():
                 shutil.move(path, os.path.join(atomic_dir.work_dir, os.path.basename(path)))
-        return Fingerprint(algorithm=fingerprint.algorithm, hash=fingerprint)
+        return Fingerprint.from_hashing_fingerprint(fingerprint)
 
     @staticmethod
     def _create_file_artifact(
@@ -98,7 +94,6 @@ def _create_file_artifact(
     def _download(
         self,
         url,  # type: str
-        requirement,  # type: ParsedRequirement
         download_dir,  # type: str
     ):
         # type: (...) -> Job
@@ -113,19 +108,9 @@ def _download(
         # care about wheel tags, environment markers or Requires-Python. The locker's download
         # observer does just this for universal locks with no target system or requires python
         # restrictions.
-        download_observer = locker.patch(
-            root_requirements=[requirement],
-            pip_version=self.package_index_configuration.pip_version,
-            resolver=self.resolver,
-            lock_configuration=LockConfiguration(style=LockStyle.UNIVERSAL),
-            download_dir=download_dir,
-            fingerprint_service=FingerprintService.create(
-                url_fetcher=URLFetcher(
-                    network_configuration=self.package_index_configuration.network_configuration,
-                    password_entries=self.package_index_configuration.password_entries,
-                ),
-                max_parallel_jobs=self.max_parallel_jobs,
-            ),
+        download_observer = DownloadObserver(
+            analyzer=None,
+            patch=locker.patch(lock_configuration=LockConfiguration(style=LockStyle.UNIVERSAL)),
         )
         return self.pip.spawn_download_distributions(
             download_dir=download_dir,
@@ -153,17 +138,15 @@ def _download_and_fingerprint(self, url):
             )
 
         return SpawnedJob.and_then(
-            self._download(
-                url=url, requirement=parse_requirement_string(url), download_dir=download_dir
-            ),
+            self._download(url=url, download_dir=download_dir),
             result_func=lambda: self._create_file_artifact(
                 url, fingerprint=self._fingerprint_and_move(temp_dest), verified=True
             ),
         )
 
     def _to_file_artifact(self, artifact):
         # type: (PartialArtifact) -> SpawnedJob[FileArtifact]
-        url = artifact.url
+        url = artifact.url.normalized_url
         fingerprint = artifact.fingerprint
         if fingerprint:
             return SpawnedJob.completed(
@@ -182,7 +165,6 @@ def fingerprint(self, artifacts):
     def download(
         self,
         artifact,  # type: FileArtifact
-        project_name,  # type: ProjectName
         dest_dir,  # type: str
         digest,  # type: HintedDigest
     ):
@@ -198,13 +180,7 @@ def download(
                 return Error(str(e))
         else:
             try:
-                self._download(
-                    url=artifact.url,
-                    requirement=parse_requirement_string(
-                        artifact.as_unparsed_requirement(project_name=project_name)
-                    ),
-                    download_dir=dest_dir,
-                ).wait()
+                self._download(url=artifact.url, download_dir=dest_dir).wait()
             except Job.Error as e:
                 return Error((e.stderr or str(e)).splitlines()[-1])
         hashing.file_hash(dest_file, digest)

pex/resolve/lock_resolver.py

@@ -66,9 +66,7 @@ def save(
         digest,  # type: HintedDigest
     ):
         # type: (...) -> Union[str, Error]
-        return self._downloader.download(
-            artifact=artifact, project_name=project_name, dest_dir=dest_dir, digest=digest
-        )
+        return self._downloader.download(artifact=artifact, dest_dir=dest_dir, digest=digest)
 
 
 class VCSArtifactDownloadManager(DownloadManager[VCSArtifact]):

pex/resolve/locked_resolve.py

@@ -16,7 +16,13 @@
 from pex.pep_503 import ProjectName
 from pex.rank import Rank
 from pex.requirements import VCS, VCSScheme, parse_scheme
-from pex.resolve.resolved_requirement import Fingerprint, PartialArtifact, Pin, ResolvedRequirement
+from pex.resolve.resolved_requirement import (
+    ArtifactURL,
+    Fingerprint,
+    PartialArtifact,
+    Pin,
+    ResolvedRequirement,
+)
 from pex.result import Error
 from pex.sorted_tuple import SortedTuple
 from pex.targets import LocalInterpreter, Target
@@ -95,47 +101,53 @@ def _validate_only_set_for_universal(
 @attr.s(frozen=True)
 class Artifact(object):
     @classmethod
-    def from_url(
+    def from_artifact_url(
         cls,
-        url,  # type: str
+        artifact_url,  # type: ArtifactURL
         fingerprint,  # type: Fingerprint
         verified=False,  # type: bool
     ):
         # type: (...) -> Union[FileArtifact, LocalProjectArtifact, VCSArtifact]
-        url_info = urlparse.urlparse(url)
-        parsed_scheme = parse_scheme(url_info.scheme)
-        if isinstance(parsed_scheme, VCSScheme):
-            return VCSArtifact(
-                url=url, fingerprint=fingerprint, verified=verified, vcs=parsed_scheme.vcs
+        if isinstance(artifact_url.scheme, VCSScheme):
+            return VCSArtifact.from_artifact_url(
+                artifact_url=artifact_url,
+                fingerprint=fingerprint,
+                verified=verified,
             )
 
-        path = unquote(url_info.path)
-        if "file" == parsed_scheme and os.path.isdir(path):
-            directory = os.path.normpath(path)
+        if "file" == artifact_url.scheme and os.path.isdir(artifact_url.path):
+            directory = os.path.normpath(artifact_url.path)
             return LocalProjectArtifact(
-                url=url, fingerprint=fingerprint, verified=verified, directory=directory
+                url=artifact_url.normalized_url,
+                fingerprint=fingerprint,
+                verified=verified,
+                directory=directory,
             )
 
-        filename = os.path.basename(path)
-        return FileArtifact(url=url, fingerprint=fingerprint, verified=verified, filename=filename)
+        filename = os.path.basename(artifact_url.path)
+        return FileArtifact(
+            url=artifact_url.normalized_url,
+            fingerprint=fingerprint,
+            verified=verified,
+            filename=filename,
+        )
+
+    @classmethod
+    def from_url(
+        cls,
+        url,  # type: str
+        fingerprint,  # type: Fingerprint
+        verified=False,  # type: bool
+    ):
+        # type: (...) -> Union[FileArtifact, LocalProjectArtifact, VCSArtifact]
+        return cls.from_artifact_url(
+            artifact_url=ArtifactURL.parse(url), fingerprint=fingerprint, verified=verified
+        )
 
     url = attr.ib()  # type: str
     fingerprint = attr.ib()  # type: Fingerprint
     verified = attr.ib()  # type: bool
 
-    def as_unparsed_requirement(self, project_name):
-        # type: (ProjectName) -> str
-        url_info = urlparse.urlparse(self.url)
-        if url_info.fragment:
-            fragment_parameters = urlparse.parse_qs(url_info.fragment)
-            names = fragment_parameters.get("egg")
-            if names and ProjectName(names[-1]) == project_name:
-                # A Pip proprietary URL requirement.
-                return self.url
-        # A PEP-440 direct reference URL requirement with the project name stripped from earlier
-        # processing. See: https://peps.python.org/pep-0440/#direct-references
-        return "{project_name} @ {url}".format(project_name=project_name, url=self.url)
-
 
 @attr.s(frozen=True)
 class FileArtifact(Artifact):
@@ -165,12 +177,48 @@ def is_source(self):
 
 @attr.s(frozen=True)
 class VCSArtifact(Artifact):
+    @classmethod
+    def from_artifact_url(
+        cls,
+        artifact_url,  # type: ArtifactURL
+        fingerprint,  # type: Fingerprint
+        verified=False,  # type: bool
+    ):
+        # type: (...) -> VCSArtifact
+        if not isinstance(artifact_url.scheme, VCSScheme):
+            raise ValueError(
+                "The given artifact URL is not that of a VCS artifact: {url}".format(
+                    url=artifact_url.raw_url
+                )
+            )
+        return cls(
+            # N.B.: We need the raw URL in order to have access to the fragment needed for
+            # `as_unparsed_requirement`.
+            url=artifact_url.raw_url,
+            fingerprint=fingerprint,
+            verified=verified,
+            vcs=artifact_url.scheme.vcs,
+        )
+
     vcs = attr.ib()  # type: VCS.Value
 
     @property
     def is_source(self):
         return True
 
+    def as_unparsed_requirement(self, project_name):
+        # type: (ProjectName) -> str
+        url_info = urlparse.urlparse(self.url)
+        if url_info.fragment:
+            fragment_parameters = urlparse.parse_qs(url_info.fragment)
+            names = fragment_parameters.get("egg")
+            if names and ProjectName(names[-1]) == project_name:
+                # A Pip proprietary VCS requirement.
+                return self.url
+        # A PEP-440 direct reference VCS requirement with the project name stripped from earlier
+        # processing. See: https://peps.python.org/pep-0440/#direct-references
+        return "{project_name} @ {url}".format(project_name=project_name, url=self.url)
+
 
 @attr.s(frozen=True)
 class RankedArtifact(object):
@@ -447,8 +495,8 @@ def resolve_fingerprint(partial_artifact):
                     to_map="\n".join(map(str, artifacts_to_fingerprint)),
                 )
             )
-            return Artifact.from_url(
-                url=partial_artifact.url,
+            return Artifact.from_artifact_url(
+                artifact_url=partial_artifact.url,
                 fingerprint=partial_artifact.fingerprint,
                 verified=partial_artifact.verified,
             )