Get Buck working for 3.10 (#2696)

Summary:
Pull Request resolved: #2696

On 3.10 some `collections` classes have been moved to
`collections.abc` and it causes Buck 1 to crash. This diff fixes those
instances.

Really pex should be upgraded, but I made an attempt and it was very difficult
due to our custom changes to pex.

fbshipit-source-id: daca9d9d35fbb92a9faebca2890d5e2cf78dd67b

scripts/slice_trace.py

@@ -16,7 +16,6 @@
 
 import argparse
 import codecs
-import collections
 import math
 import os
 

third-party/py/pathlib/test_pathlib.py

@@ -1,4 +1,3 @@
-import collections
 import io
 import os
 import errno
@@ -19,8 +18,10 @@
         raise ImportError("unittest2 is required for tests on pre-2.7")
 
 try:
+    import collections.abc as collections_abc
     from test import support
 except ImportError:
+    import collections as collections_abc  # Fallback for  PY3.2.
     from test import test_support as support
 TESTFN = support.TESTFN
 
@@ -1395,7 +1396,7 @@ def _check(glob, expected):
         P = self.cls
         p = P(BASE)
         it = p.glob("fileA")
-        self.assertIsInstance(it, collections.Iterator)
+        self.assertIsInstance(it, collections_abc.Iterator)
         _check(it, ["fileA"])
         _check(p.glob("fileB"), [])
         _check(p.glob("dir*/file*"), ["dirB/fileB", "dirC/fileC"])
@@ -1420,7 +1421,7 @@ def _check(glob, expected):
         P = self.cls
         p = P(BASE)
         it = p.rglob("fileA")
-        self.assertIsInstance(it, collections.Iterator)
+        self.assertIsInstance(it, collections_abc.Iterator)
         # XXX cannot test because of symlink loops in the test setup
         #_check(it, ["fileA"])
         #_check(p.rglob("fileB"), ["dirB/fileB"])

third-party/py/pex/README.facebook

@@ -16,3 +16,5 @@ Local modifications:
  - Make pexes work when being invoked from a version of python without site.USER_SITE
  - Fix concurrency when two instances of zip-safe pex invoked D21508334
  - Handle EACCES and EPERM in safe_copy
+ - Imported from collections.abc instead of collections to support Python 3.10
+ - Back-ported removal of MarkerEvaluation from pieces of commit ba5633b3c7b9317b87130a2ea671d8c008a673d6 and a718819d2849196e902808301c9a95724510c5c1

third-party/py/pex/pex/base.py

@@ -3,7 +3,10 @@
 
 from __future__ import absolute_import
 
-from collections import Iterable
+try:
+    from collections.abc import Iterable
+except ImportError:  # For PY3.2
+    from collections import Iterable
 
 from pkg_resources import Requirement
 

third-party/py/pex/pex/link.py

@@ -5,12 +5,17 @@
 
 import os
 import posixpath
-from collections import Iterable
 
 from .compatibility import string as compatible_string
 from .compatibility import PY3
 from .util import Memoizer
 
+
+try:
+    from collections.abc import Iterable
+except ImportError:  # For PY3.2
+    from collections import Iterable
+
 if PY3:
   import urllib.parse as urlparse
 else:

third-party/py/pex/pex/orderedset.py

@@ -8,10 +8,14 @@
 # modifications
 #
 
-import collections
 
+try:
+    import collections.abc as collections_abc
+except ImportError:  # For PY3.2
+    import collections as collections_abc
 
-class OrderedSet(collections.MutableSet):
+
+class OrderedSet(collections_abc.MutableSet):
   KEY, PREV, NEXT = range(3)
 
   def __init__(self, iterable=None):

third-party/py/pywatchman/pywatchman/pybser.py

@@ -32,7 +32,6 @@
 # no unicode literals
 
 import binascii
-import collections
 import ctypes
 import struct
 import sys
@@ -41,6 +40,11 @@
     compat,
 )
 
+try:
+    import collections.abc as collections_abc
+except ImportError:
+    import collections as collections_abc  # Fallback for PY3.2.
+
 BSER_ARRAY = b'\x00'
 BSER_OBJECT = b'\x01'
 BSER_BYTESTRING = b'\x02'
@@ -177,8 +181,8 @@ def append_recursive(self, val):
             self.ensure_size(needed)
             struct.pack_into(b'=cd', self.buf, self.wpos, BSER_REAL, val)
             self.wpos += needed
-        elif isinstance(val, collections.Mapping) and \
-            isinstance(val, collections.Sized):
+        elif isinstance(val, collections_abc.Mapping) and \
+            isinstance(val, collections_abc.Sized):
             val_len = len(val)
             size = _int_size(val_len)
             needed = 2 + size
@@ -205,8 +209,8 @@ def append_recursive(self, val):
             for k, v in iteritems:
                 self.append_string(k)
                 self.append_recursive(v)
-        elif isinstance(val, collections.Iterable) and \
-            isinstance(val, collections.Sized):
+        elif isinstance(val, collections_abc.Iterable) and \
+            isinstance(val, collections_abc.Sized):
             val_len = len(val)
             size = _int_size(val_len)
             needed = 2 + size

third-party/py/pywatchman/tests/tests.py

@@ -6,7 +6,6 @@
 # no unicode literals
 
 import binascii
-import collections
 import inspect
 import unittest
 import os

third-party/py/setuptools/README.facebook

@@ -1,4 +1,4 @@
 Project URL: https://pypi.python.org/packages/source/s/setuptools/setuptools-18.5.tar.gz#md5=533c868f01169a3085177dffe5e768bb
 Version: 18.5
 License: PSF or ZPL
-Local modifications: kept only pkg_resources & _markerlib modules
+Local modifications: kept only pkg_resources & _markerlib modules0

third-party/py/setuptools/pkg_resources/__init__.py

@@ -29,7 +29,6 @@
 import functools
 import pkgutil
 import token
-import symbol
 import operator
 import platform
 import collections
@@ -1402,202 +1401,30 @@ def to_filename(name):
     return name.replace('-','_')
 
 
-class MarkerEvaluation(object):
-    values = {
-        'os_name': lambda: os.name,
-        'sys_platform': lambda: sys.platform,
-        'python_full_version': platform.python_version,
-        'python_version': lambda: platform.python_version()[:3],
-        'platform_version': platform.version,
-        'platform_machine': platform.machine,
-        'platform_python_implementation': platform.python_implementation,
-        'python_implementation': platform.python_implementation,
-    }
-
-    @classmethod
-    def is_invalid_marker(cls, text):
-        """
-        Validate text as a PEP 426 environment marker; return an exception
-        if invalid or False otherwise.
-        """
-        try:
-            cls.evaluate_marker(text)
-        except SyntaxError as e:
-            return cls.normalize_exception(e)
-        return False
-
-    @staticmethod
-    def normalize_exception(exc):
-        """
-        Given a SyntaxError from a marker evaluation, normalize the error
-        message:
-         - Remove indications of filename and line number.
-         - Replace platform-specific error messages with standard error
-           messages.
-        """
-        subs = {
-            'unexpected EOF while parsing': 'invalid syntax',
-            'parenthesis is never closed': 'invalid syntax',
-        }
-        exc.filename = None
-        exc.lineno = None
-        exc.msg = subs.get(exc.msg, exc.msg)
-        return exc
-
-    @classmethod
-    def and_test(cls, nodelist):
-        # MUST NOT short-circuit evaluation, or invalid syntax can be skipped!
-        items = [
-            cls.interpret(nodelist[i])
-            for i in range(1, len(nodelist), 2)
-        ]
-        return functools.reduce(operator.and_, items)
-
-    @classmethod
-    def test(cls, nodelist):
-        # MUST NOT short-circuit evaluation, or invalid syntax can be skipped!
-        items = [
-            cls.interpret(nodelist[i])
-            for i in range(1, len(nodelist), 2)
-        ]
-        return functools.reduce(operator.or_, items)
-
-    @classmethod
-    def atom(cls, nodelist):
-        t = nodelist[1][0]
-        if t == token.LPAR:
-            if nodelist[2][0] == token.RPAR:
-                raise SyntaxError("Empty parentheses")
-            return cls.interpret(nodelist[2])
-        msg = "Language feature not supported in environment markers"
-        raise SyntaxError(msg)
-
-    @classmethod
-    def comparison(cls, nodelist):
-        if len(nodelist) > 4:
-            msg = "Chained comparison not allowed in environment markers"
-            raise SyntaxError(msg)
-        comp = nodelist[2][1]
-        cop = comp[1]
-        if comp[0] == token.NAME:
-            if len(nodelist[2]) == 3:
-                if cop == 'not':
-                    cop = 'not in'
-                else:
-                    cop = 'is not'
-        try:
-            cop = cls.get_op(cop)
-        except KeyError:
-            msg = repr(cop) + " operator not allowed in environment markers"
-            raise SyntaxError(msg)
-        return cop(cls.evaluate(nodelist[1]), cls.evaluate(nodelist[3]))
-
-    @classmethod
-    def get_op(cls, op):
-        ops = {
-            symbol.test: cls.test,
-            symbol.and_test: cls.and_test,
-            symbol.atom: cls.atom,
-            symbol.comparison: cls.comparison,
-            'not in': lambda x, y: x not in y,
-            'in': lambda x, y: x in y,
-            '==': operator.eq,
-            '!=': operator.ne,
-            '<':  operator.lt,
-            '>':  operator.gt,
-            '<=': operator.le,
-            '>=': operator.ge,
-        }
-        if hasattr(symbol, 'or_test'):
-            ops[symbol.or_test] = cls.test
-        return ops[op]
-
-    @classmethod
-    def evaluate_marker(cls, text, extra=None):
-        """
-        Evaluate a PEP 426 environment marker on CPython 2.4+.
-        Return a boolean indicating the marker result in this environment.
-        Raise SyntaxError if marker is invalid.
-
-        This implementation uses the 'parser' module, which is not implemented
-        on
-        Jython and has been superseded by the 'ast' module in Python 2.6 and
-        later.
-        """
-        return cls.interpret(parser.expr(text).totuple(1)[1])
-
-    @staticmethod
-    def _translate_metadata2(env):
-        """
-        Markerlib implements Metadata 1.2 (PEP 345) environment markers.
-        Translate the variables to Metadata 2.0 (PEP 426).
-        """
-        return dict(
-            (key.replace('.', '_'), value)
-            for key, value in env
-        )
-
-    @classmethod
-    def _markerlib_evaluate(cls, text):
-        """
-        Evaluate a PEP 426 environment marker using markerlib.
-        Return a boolean indicating the marker result in this environment.
-        Raise SyntaxError if marker is invalid.
-        """
-        import _markerlib
-
-        env = cls._translate_metadata2(_markerlib.default_environment())
-        try:
-            result = _markerlib.interpret(text, env)
-        except NameError as e:
-            raise SyntaxError(e.args[0])
-        return result
-
-    if 'parser' not in globals():
-        # Fall back to less-complete _markerlib implementation if 'parser' module
-        # is not available.
-        evaluate_marker = _markerlib_evaluate
-
-    @classmethod
-    def interpret(cls, nodelist):
-        while len(nodelist)==2: nodelist = nodelist[1]
-        try:
-            op = cls.get_op(nodelist[0])
-        except KeyError:
-            raise SyntaxError("Comparison or logical expression expected")
-        return op(nodelist)
+def invalid_marker(text):
+    """
+    Validate text as a PEP 508 environment marker; return an exception
+    if invalid or False otherwise.
+    """
+    try:
+        evaluate_marker(text)
+    except packaging.markers.InvalidMarker as e:
+        e.filename = None
+        e.lineno = None
+        return e
+    return False
 
-    @classmethod
-    def evaluate(cls, nodelist):
-        while len(nodelist)==2: nodelist = nodelist[1]
-        kind = nodelist[0]
-        name = nodelist[1]
-        if kind==token.NAME:
-            try:
-                op = cls.values[name]
-            except KeyError:
-                raise SyntaxError("Unknown name %r" % name)
-            return op()
-        if kind==token.STRING:
-            s = nodelist[1]
-            if not cls._safe_string(s):
-                raise SyntaxError(
-                    "Only plain strings allowed in environment markers")
-            return s[1:-1]
-        msg = "Language feature not supported in environment markers"
-        raise SyntaxError(msg)
 
-    @staticmethod
-    def _safe_string(cand):
-        return (
-            cand[:1] in "'\"" and
-            not cand.startswith('"""') and
-            not cand.startswith("'''") and
-            '\\' not in cand
-        )
+def evaluate_marker(text, extra=None):
+    """
+    Evaluate a PEP 508 environment marker.
+    Return a boolean indicating the marker result in this environment.
+    Raise InvalidMarker if marker is invalid.
+    This implementation uses the 'pyparsing' module.
+    """
+    marker = packaging.markers.Marker(text)
+    return marker.evaluate()
 
-invalid_marker = MarkerEvaluation.is_invalid_marker
-evaluate_marker = MarkerEvaluation.evaluate_marker
 
 class NullProvider:
     """Try to implement resources and metadata for arbitrary PEP 302 loaders"""