Handle locking all direct reference URL forms.

Previously, it was assumed that direct reference URLs were either VCS
URLs or else wheel or sdist URLs. This neglected two remaining cases,
both of which failed to lock with better or worse error messages. The
two missed cases were:
1. URLs of source archives not conforming to the sdist quasi-standard
   naming convention of `<project name>-<version>.{.tar.gz,.zip}`.
2. Local file:// URLs pointing at project directories.

A notable case of the 1st are project archives provided by GitHub.
A notable need for the 2nd case comes from Pants where Pip proprietary
requirement strings are not handled (e.g.: `/path/to/project`) and a
direct reference URL must be used instead (e.g.: `projectname @
/path/to/project`).

Fixes pex-tool#2057

pex/common.py

@@ -213,8 +213,9 @@ def _chmod(self, info, path):
         # This magic works to extract perm bits from the 32 bit external file attributes field for
         # unix-created zip files, for the layout, see:
         #   https://www.forensicswiki.org/wiki/ZIP#External_file_attributes
-        attr = info.external_attr >> 16
-        os.chmod(path, attr)
+        if info.external_attr > 0xFFFF:
+            attr = info.external_attr >> 16
+            os.chmod(path, attr)
 
 
 @contextlib.contextmanager

pex/dist_metadata.py

@@ -291,8 +291,10 @@ def from_filename(cls, path):
         # characters and dashes.
         fname = _strip_sdist_path(path)
         if fname is not None:
-            project_name, version = fname.rsplit("-", 1)
-            return cls(project_name=project_name, version=version)
+            components = fname.rsplit("-", 1)
+            if len(components) == 2:
+                project_name, version = components
+                return cls(project_name=project_name, version=version)
 
         raise UnrecognizedDistributionFormat(
             "The distribution at path {!r} does not have a file name matching known sdist or wheel "

pex/resolve/downloads.py

@@ -10,6 +10,7 @@
 from pex.fetcher import URLFetcher
 from pex.hashing import Sha256
 from pex.jobs import Job, Raise, SpawnedJob, execute_parallel
+from pex.pep_503 import ProjectName
 from pex.pip.installation import get_pip
 from pex.pip.tool import PackageIndexConfiguration, Pip
 from pex.requirements import parse_requirement_string
@@ -29,6 +30,7 @@
     import attr  # vendor:skip
 
     from pex.hashing import HintedDigest
+    from pex.requirements import ParsedRequirement
 else:
     from pex.third_party import attr
 
@@ -96,6 +98,7 @@ def _create_file_artifact(
     def _download(
         self,
         url,  # type: str
+        requirement,  # type: ParsedRequirement
         download_dir,  # type: str
     ):
         # type: (...) -> Job
@@ -111,7 +114,7 @@ def _download(
         # observer does just this for universal locks with no target system or requires python
         # restrictions.
         download_observer = locker.patch(
-            root_requirements=[parse_requirement_string(url)],
+            root_requirements=[requirement],
             pip_version=self.package_index_configuration.pip_version,
             resolver=self.resolver,
             lock_configuration=LockConfiguration(style=LockStyle.UNIVERSAL),
@@ -150,7 +153,9 @@ def _download_and_fingerprint(self, url):
             )
 
         return SpawnedJob.and_then(
-            self._download(url=url, download_dir=download_dir),
+            self._download(
+                url=url, requirement=parse_requirement_string(url), download_dir=download_dir
+            ),
             result_func=lambda: self._create_file_artifact(
                 url, fingerprint=self._fingerprint_and_move(temp_dest), verified=True
             ),
@@ -177,6 +182,7 @@ def fingerprint(self, artifacts):
     def download(
         self,
         artifact,  # type: FileArtifact
+        project_name,  # type: ProjectName
         dest_dir,  # type: str
         digest,  # type: HintedDigest
     ):
@@ -192,7 +198,13 @@ def download(
                 return Error(str(e))
         else:
             try:
-                self._download(url=artifact.url, download_dir=dest_dir).wait()
+                self._download(
+                    url=artifact.url,
+                    requirement=parse_requirement_string(
+                        artifact.as_unparsed_requirement(project_name=project_name)
+                    ),
+                    download_dir=dest_dir,
+                ).wait()
             except Job.Error as e:
                 return Error((e.stderr or str(e)).splitlines()[-1])
         hashing.file_hash(dest_file, digest)

pex/resolve/lock_resolver.py

@@ -66,7 +66,9 @@ def save(
         digest,  # type: HintedDigest
     ):
         # type: (...) -> Union[str, Error]
-        return self._downloader.download(artifact=artifact, dest_dir=dest_dir, digest=digest)
+        return self._downloader.download(
+            artifact=artifact, project_name=project_name, dest_dir=dest_dir, digest=digest
+        )
 
 
 class VCSArtifactDownloadManager(DownloadManager[VCSArtifact]):

pex/resolve/locked_resolve.py

@@ -123,6 +123,19 @@ def from_url(
     fingerprint = attr.ib()  # type: Fingerprint
     verified = attr.ib()  # type: bool
 
+    def as_unparsed_requirement(self, project_name):
+        # type: (ProjectName) -> str
+        url_info = urlparse.urlparse(self.url)
+        if url_info.fragment:
+            fragment_parameters = urlparse.parse_qs(url_info.fragment)
+            names = fragment_parameters.get("egg")
+            if names and ProjectName(names[-1]) == project_name:
+                # A Pip proprietary URL requirement.
+                return self.url
+        # A PEP-440 direct reference URL requirement with the project name stripped from earlier
+        # processing. See: https://peps.python.org/pep-0440/#direct-references
+        return "{project_name} @ {url}".format(project_name=project_name, url=self.url)
+
 
 @attr.s(frozen=True)
 class FileArtifact(Artifact):
@@ -158,19 +171,6 @@ class VCSArtifact(Artifact):
     def is_source(self):
         return True
 
-    def as_unparsed_requirement(self, project_name):
-        # type: (ProjectName) -> str
-        url_info = urlparse.urlparse(self.url)
-        if url_info.fragment:
-            fragment_parameters = urlparse.parse_qs(url_info.fragment)
-            names = fragment_parameters.get("egg")
-            if names and ProjectName(names[-1]) == project_name:
-                # A Pip proprietary VCS requirement.
-                return self.url
-        # A PEP-440 direct reference VCS requirement with the project name stripped from earlier
-        # processing. See: https://peps.python.org/pep-0440/#direct-references
-        return "{project_name} @ {url}".format(project_name=project_name, url=self.url)
-
 
 @attr.s(frozen=True)
 class RankedArtifact(object):