diff --git a/scripts/slice_trace.py b/scripts/slice_trace.py index fcd2343b346..e1ffc925ed8 100644 --- a/scripts/slice_trace.py +++ b/scripts/slice_trace.py @@ -16,8 +16,6 @@ import argparse import codecs -import collections -import json import math import os diff --git a/third-party/py/pathlib/test_pathlib.py b/third-party/py/pathlib/test_pathlib.py index 7147e4ee5a3..5522fdcc34d 100755 --- a/third-party/py/pathlib/test_pathlib.py +++ b/third-party/py/pathlib/test_pathlib.py @@ -1,4 +1,3 @@ -import collections import io import os import errno @@ -19,8 +18,10 @@ raise ImportError("unittest2 is required for tests on pre-2.7") try: + import collections.abc as collections_abc from test import support except ImportError: + import collections as collections_abc # Fallback for PY3.2. from test import test_support as support TESTFN = support.TESTFN @@ -1395,7 +1396,7 @@ def _check(glob, expected): P = self.cls p = P(BASE) it = p.glob("fileA") - self.assertIsInstance(it, collections.Iterator) + self.assertIsInstance(it, collections_abc.Iterator) _check(it, ["fileA"]) _check(p.glob("fileB"), []) _check(p.glob("dir*/file*"), ["dirB/fileB", "dirC/fileC"]) @@ -1420,7 +1421,7 @@ def _check(glob, expected): P = self.cls p = P(BASE) it = p.rglob("fileA") - self.assertIsInstance(it, collections.Iterator) + self.assertIsInstance(it, collections_abc.Iterator) # XXX cannot test because of symlink loops in the test setup #_check(it, ["fileA"]) #_check(p.rglob("fileB"), ["dirB/fileB"]) diff --git a/third-party/py/pex/README.facebook b/third-party/py/pex/README.facebook index fd7aee556fc..f08e6b58844 100644 --- a/third-party/py/pex/README.facebook +++ b/third-party/py/pex/README.facebook @@ -13,3 +13,5 @@ Local modifications: - Back-ported Python 3.6 compatibility commit c5ab73fd4d8e816e21a89d48c8d0c8095ef5a49c - Back-ported namespaced packages fix, commit 7d2dc7f500aa7ae227c3ddca4b278b807d353a5e - Fixed Python 3 issue with writing bytes to a text file (`with open(path, 'wb') as fp:` on line 68 in `compiler.py`) + - Imported from collections.abc instead of collections to support Python 3.10 + - Back-ported removal of MarkerEvaluation from pieces of commit ba5633b3c7b9317b87130a2ea671d8c008a673d6 and a718819d2849196e902808301c9a95724510c5c1 diff --git a/third-party/py/pex/pex/base.py b/third-party/py/pex/pex/base.py index 7d90443dd3c..2a3d775f9e0 100644 --- a/third-party/py/pex/pex/base.py +++ b/third-party/py/pex/pex/base.py @@ -3,7 +3,10 @@ from __future__ import absolute_import -from collections import Iterable +try: + from collections.abc import Iterable +except ImportError: # For PY3.2 + from collections import Iterable from pkg_resources import Requirement diff --git a/third-party/py/pex/pex/link.py b/third-party/py/pex/pex/link.py index 542f344e09a..5815adc94d2 100644 --- a/third-party/py/pex/pex/link.py +++ b/third-party/py/pex/pex/link.py @@ -5,12 +5,17 @@ import os import posixpath -from collections import Iterable from .compatibility import string as compatible_string from .compatibility import PY3 from .util import Memoizer + +try: + from collections.abc import Iterable +except ImportError: # For PY3.2 + from collections import Iterable + if PY3: import urllib.parse as urlparse else: diff --git a/third-party/py/pex/pex/orderedset.py b/third-party/py/pex/pex/orderedset.py index 3eb5cb45414..1a8db948504 100644 --- a/third-party/py/pex/pex/orderedset.py +++ b/third-party/py/pex/pex/orderedset.py @@ -8,10 +8,14 @@ # modifications # -import collections +try: + import collections.abc as collections_abc +except ImportError: # For PY3.2 + import collections as collections_abc -class OrderedSet(collections.MutableSet): + +class OrderedSet(collections_abc.MutableSet): KEY, PREV, NEXT = range(3) def __init__(self, iterable=None): diff --git a/third-party/py/pywatchman/pywatchman/pybser.py b/third-party/py/pywatchman/pywatchman/pybser.py index 91dff190c03..01f0c42bc0a 100644 --- a/third-party/py/pywatchman/pywatchman/pybser.py +++ b/third-party/py/pywatchman/pywatchman/pybser.py @@ -32,7 +32,6 @@ # no unicode literals import binascii -import collections import ctypes import struct import sys @@ -41,6 +40,11 @@ compat, ) +try: + import collections.abc as collections_abc +except ImportError: + import collections as collections_abc # Fallback for PY3.2. + BSER_ARRAY = b'\x00' BSER_OBJECT = b'\x01' BSER_BYTESTRING = b'\x02' @@ -177,8 +181,8 @@ def append_recursive(self, val): self.ensure_size(needed) struct.pack_into(b'=cd', self.buf, self.wpos, BSER_REAL, val) self.wpos += needed - elif isinstance(val, collections.Mapping) and \ - isinstance(val, collections.Sized): + elif isinstance(val, collections_abc.Mapping) and \ + isinstance(val, collections_abc.Sized): val_len = len(val) size = _int_size(val_len) needed = 2 + size @@ -205,8 +209,8 @@ def append_recursive(self, val): for k, v in iteritems: self.append_string(k) self.append_recursive(v) - elif isinstance(val, collections.Iterable) and \ - isinstance(val, collections.Sized): + elif isinstance(val, collections_abc.Iterable) and \ + isinstance(val, collections_abc.Sized): val_len = len(val) size = _int_size(val_len) needed = 2 + size diff --git a/third-party/py/pywatchman/tests/tests.py b/third-party/py/pywatchman/tests/tests.py index 02d9a0b6396..7e77da43ede 100755 --- a/third-party/py/pywatchman/tests/tests.py +++ b/third-party/py/pywatchman/tests/tests.py @@ -6,7 +6,6 @@ # no unicode literals import binascii -import collections import inspect import unittest import os diff --git a/third-party/py/setuptools/pkg_resources/__init__.py b/third-party/py/setuptools/pkg_resources/__init__.py index d09e0b6f9af..262399db8e3 100644 --- a/third-party/py/setuptools/pkg_resources/__init__.py +++ b/third-party/py/setuptools/pkg_resources/__init__.py @@ -29,7 +29,6 @@ import functools import pkgutil import token -import symbol import operator import platform import collections @@ -1402,202 +1401,30 @@ def to_filename(name): return name.replace('-','_') -class MarkerEvaluation(object): - values = { - 'os_name': lambda: os.name, - 'sys_platform': lambda: sys.platform, - 'python_full_version': platform.python_version, - 'python_version': lambda: platform.python_version()[:3], - 'platform_version': platform.version, - 'platform_machine': platform.machine, - 'platform_python_implementation': platform.python_implementation, - 'python_implementation': platform.python_implementation, - } - - @classmethod - def is_invalid_marker(cls, text): - """ - Validate text as a PEP 426 environment marker; return an exception - if invalid or False otherwise. - """ - try: - cls.evaluate_marker(text) - except SyntaxError as e: - return cls.normalize_exception(e) - return False - - @staticmethod - def normalize_exception(exc): - """ - Given a SyntaxError from a marker evaluation, normalize the error - message: - - Remove indications of filename and line number. - - Replace platform-specific error messages with standard error - messages. - """ - subs = { - 'unexpected EOF while parsing': 'invalid syntax', - 'parenthesis is never closed': 'invalid syntax', - } - exc.filename = None - exc.lineno = None - exc.msg = subs.get(exc.msg, exc.msg) - return exc - - @classmethod - def and_test(cls, nodelist): - # MUST NOT short-circuit evaluation, or invalid syntax can be skipped! - items = [ - cls.interpret(nodelist[i]) - for i in range(1, len(nodelist), 2) - ] - return functools.reduce(operator.and_, items) - - @classmethod - def test(cls, nodelist): - # MUST NOT short-circuit evaluation, or invalid syntax can be skipped! - items = [ - cls.interpret(nodelist[i]) - for i in range(1, len(nodelist), 2) - ] - return functools.reduce(operator.or_, items) - - @classmethod - def atom(cls, nodelist): - t = nodelist[1][0] - if t == token.LPAR: - if nodelist[2][0] == token.RPAR: - raise SyntaxError("Empty parentheses") - return cls.interpret(nodelist[2]) - msg = "Language feature not supported in environment markers" - raise SyntaxError(msg) - - @classmethod - def comparison(cls, nodelist): - if len(nodelist) > 4: - msg = "Chained comparison not allowed in environment markers" - raise SyntaxError(msg) - comp = nodelist[2][1] - cop = comp[1] - if comp[0] == token.NAME: - if len(nodelist[2]) == 3: - if cop == 'not': - cop = 'not in' - else: - cop = 'is not' - try: - cop = cls.get_op(cop) - except KeyError: - msg = repr(cop) + " operator not allowed in environment markers" - raise SyntaxError(msg) - return cop(cls.evaluate(nodelist[1]), cls.evaluate(nodelist[3])) - - @classmethod - def get_op(cls, op): - ops = { - symbol.test: cls.test, - symbol.and_test: cls.and_test, - symbol.atom: cls.atom, - symbol.comparison: cls.comparison, - 'not in': lambda x, y: x not in y, - 'in': lambda x, y: x in y, - '==': operator.eq, - '!=': operator.ne, - '<': operator.lt, - '>': operator.gt, - '<=': operator.le, - '>=': operator.ge, - } - if hasattr(symbol, 'or_test'): - ops[symbol.or_test] = cls.test - return ops[op] - - @classmethod - def evaluate_marker(cls, text, extra=None): - """ - Evaluate a PEP 426 environment marker on CPython 2.4+. - Return a boolean indicating the marker result in this environment. - Raise SyntaxError if marker is invalid. - - This implementation uses the 'parser' module, which is not implemented - on - Jython and has been superseded by the 'ast' module in Python 2.6 and - later. - """ - return cls.interpret(parser.expr(text).totuple(1)[1]) - - @staticmethod - def _translate_metadata2(env): - """ - Markerlib implements Metadata 1.2 (PEP 345) environment markers. - Translate the variables to Metadata 2.0 (PEP 426). - """ - return dict( - (key.replace('.', '_'), value) - for key, value in env - ) - - @classmethod - def _markerlib_evaluate(cls, text): - """ - Evaluate a PEP 426 environment marker using markerlib. - Return a boolean indicating the marker result in this environment. - Raise SyntaxError if marker is invalid. - """ - import _markerlib - - env = cls._translate_metadata2(_markerlib.default_environment()) - try: - result = _markerlib.interpret(text, env) - except NameError as e: - raise SyntaxError(e.args[0]) - return result - - if 'parser' not in globals(): - # Fall back to less-complete _markerlib implementation if 'parser' module - # is not available. - evaluate_marker = _markerlib_evaluate - - @classmethod - def interpret(cls, nodelist): - while len(nodelist)==2: nodelist = nodelist[1] - try: - op = cls.get_op(nodelist[0]) - except KeyError: - raise SyntaxError("Comparison or logical expression expected") - return op(nodelist) +def invalid_marker(text): + """ + Validate text as a PEP 508 environment marker; return an exception + if invalid or False otherwise. + """ + try: + evaluate_marker(text) + except packaging.markers.InvalidMarker as e: + e.filename = None + e.lineno = None + return e + return False - @classmethod - def evaluate(cls, nodelist): - while len(nodelist)==2: nodelist = nodelist[1] - kind = nodelist[0] - name = nodelist[1] - if kind==token.NAME: - try: - op = cls.values[name] - except KeyError: - raise SyntaxError("Unknown name %r" % name) - return op() - if kind==token.STRING: - s = nodelist[1] - if not cls._safe_string(s): - raise SyntaxError( - "Only plain strings allowed in environment markers") - return s[1:-1] - msg = "Language feature not supported in environment markers" - raise SyntaxError(msg) - @staticmethod - def _safe_string(cand): - return ( - cand[:1] in "'\"" and - not cand.startswith('"""') and - not cand.startswith("'''") and - '\\' not in cand - ) +def evaluate_marker(text, extra=None): + """ + Evaluate a PEP 508 environment marker. + Return a boolean indicating the marker result in this environment. + Raise InvalidMarker if marker is invalid. + This implementation uses the 'pyparsing' module. + """ + marker = packaging.markers.Marker(text) + return marker.evaluate() -invalid_marker = MarkerEvaluation.is_invalid_marker -evaluate_marker = MarkerEvaluation.evaluate_marker class NullProvider: """Try to implement resources and metadata for arbitrary PEP 302 loaders""" diff --git a/third-party/py/setuptools/pkg_resources/_vendor/packaging/markers.py b/third-party/py/setuptools/pkg_resources/_vendor/packaging/markers.py new file mode 100644 index 00000000000..80e63676661 --- /dev/null +++ b/third-party/py/setuptools/pkg_resources/_vendor/packaging/markers.py @@ -0,0 +1,221 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. +from __future__ import absolute_import, division, print_function + +import operator +import os +import platform +import sys + +from pkg_resources._vendor.pyparsing import ParseException, ParseResults, stringStart, stringEnd +from pkg_resources._vendor.pyparsing import ZeroOrMore, Group, Forward, QuotedString +from pkg_resources._vendor.pyparsing import Literal as L # noqa + +from ._compat import string_types +from .specifiers import Specifier, InvalidSpecifier + + +__all__ = [ + "InvalidMarker", "UndefinedComparison", "Marker", "default_environment", +] + + +class InvalidMarker(ValueError): + """ + An invalid marker was found, users should refer to PEP 508. + """ + + +class UndefinedComparison(ValueError): + """ + An invalid operation was attempted on a value that doesn't support it. + """ + + +class Node(object): + def __init__(self, value): + self.value = value + def __str__(self): + return str(self.value) + def __repr__(self): + return "<{0}({1!r})>".format(self.__class__.__name__, str(self)) + + +class Variable(Node): + pass + + +class Value(Node): + pass + +VARIABLE = ( + L("implementation_version") | + L("platform_python_implementation") | + L("implementation_name") | + L("python_full_version") | + L("platform_release") | + L("platform_version") | + L("platform_machine") | + L("platform_system") | + L("python_version") | + L("sys_platform") | + L("os_name") | + L("extra") +) +VARIABLE.setParseAction(lambda s, l, t: Variable(t[0])) + +VERSION_CMP = ( + L("===") | + L("==") | + L(">=") | + L("<=") | + L("!=") | + L("~=") | + L(">") | + L("<") +) +MARKER_OP = VERSION_CMP | L("not in") | L("in") +MARKER_VALUE = QuotedString("'") | QuotedString('"') +MARKER_VALUE.setParseAction(lambda s, l, t: Value(t[0])) +BOOLOP = L("and") | L("or") +MARKER_VAR = VARIABLE | MARKER_VALUE +MARKER_ITEM = Group(MARKER_VAR + MARKER_OP + MARKER_VAR) +MARKER_ITEM.setParseAction(lambda s, l, t: tuple(t[0])) +LPAREN = L("(").suppress() +RPAREN = L(")").suppress() +MARKER_EXPR = Forward() +MARKER_ATOM = MARKER_ITEM | Group(LPAREN + MARKER_EXPR + RPAREN) +MARKER_EXPR << MARKER_ATOM + ZeroOrMore(BOOLOP + MARKER_EXPR) +MARKER = stringStart + MARKER_EXPR + stringEnd + + +def _coerce_parse_result(results): + if isinstance(results, ParseResults): + return [_coerce_parse_result(i) for i in results] + else: + return results + + +def _format_marker(marker, first=True): + assert isinstance(marker, (list, tuple, string_types)) + # Sometimes we have a structure like [[...]] which is a single item list + # where the single item is itself it's own list. In that case we want skip + # the rest of this function so that we don't get extraneous () on the + # outside. + if (isinstance(marker, list) and len(marker) == 1 + and isinstance(marker[0], (list, tuple))): + return _format_marker(marker[0]) + if isinstance(marker, list): + inner = (_format_marker(m, first=False) for m in marker) + if first: + return " ".join(inner) + else: + return "(" + " ".join(inner) + ")" + elif isinstance(marker, tuple): + return '{0} {1} "{2}"'.format(*marker) + else: + return marker + + +_operators = { + "in": lambda lhs, rhs: lhs in rhs, + "not in": lambda lhs, rhs: lhs not in rhs, + "<": operator.lt, + "<=": operator.le, + "==": operator.eq, + "!=": operator.ne, + ">=": operator.ge, + ">": operator.gt, +} + + +def _eval_op(lhs, op, rhs): + try: + spec = Specifier("".join([op, rhs])) + except InvalidSpecifier: + pass + else: + return spec.contains(lhs) + oper = _operators.get(op) + if oper is None: + raise UndefinedComparison( + "Undefined {0!r} on {1!r} and {2!r}.".format(op, lhs, rhs) + ) + return oper(lhs, rhs) + + +def _evaluate_markers(markers, environment): + groups = [[]] + for marker in markers: + assert isinstance(marker, (list, tuple, string_types)) + if isinstance(marker, list): + groups[-1].append(_evaluate_markers(marker, environment)) + elif isinstance(marker, tuple): + lhs, op, rhs = marker + if isinstance(lhs, Variable): + value = _eval_op(environment[lhs.value], op, rhs.value) + else: + value = _eval_op(lhs.value, op, environment[rhs.value]) + groups[-1].append(value) + else: + assert marker in ["and", "or"] + if marker == "or": + groups.append([]) + return any(all(item) for item in groups) + + +def format_full_version(info): + version = '{0.major}.{0.minor}.{0.micro}'.format(info) + kind = info.releaselevel + if kind != 'final': + version += kind[0] + str(info.serial) + return version + + +def default_environment(): + if hasattr(sys, 'implementation'): + iver = format_full_version(sys.implementation.version) + implementation_name = sys.implementation.name + else: + iver = '0' + implementation_name = '' + return { + "implementation_name": implementation_name, + "implementation_version": iver, + "os_name": os.name, + "platform_machine": platform.machine(), + "platform_release": platform.release(), + "platform_system": platform.system(), + "platform_version": platform.version(), + "python_full_version": platform.python_version(), + "platform_python_implementation": platform.python_implementation(), + "python_version": platform.python_version()[:3], + "sys_platform": sys.platform, + } + + +class Marker(object): + def __init__(self, marker): + try: + self._markers = _coerce_parse_result(MARKER.parseString(marker)) + except ParseException: + self._markers = None + # We do this because we can't do raise ... from None in Python 2.x + if self._markers is None: + raise InvalidMarker("Invalid marker: {0!r}".format(marker)) + def __str__(self): + return _format_marker(self._markers) + def __repr__(self): + return "".format(str(self)) + def evaluate(self, environment=None): + """Evaluate a marker. + Return the boolean from evaluating the given marker against the + environment. environment is an optional argument to override all or + part of the determined environment. + The environment is determined from the current Python process. + """ + current_environment = default_environment() + if environment is not None: + current_environment.update(environment) + return _evaluate_markers(self._markers, current_environment) diff --git a/third-party/py/setuptools/pkg_resources/tests/test_markers.py b/third-party/py/setuptools/pkg_resources/tests/test_markers.py index d8844e74e3e..dec9856cffd 100644 --- a/third-party/py/setuptools/pkg_resources/tests/test_markers.py +++ b/third-party/py/setuptools/pkg_resources/tests/test_markers.py @@ -6,11 +6,6 @@ from pkg_resources import evaluate_marker -@mock.patch.dict('pkg_resources.MarkerEvaluation.values', - python_full_version=mock.Mock(return_value='2.7.10')) -def test_lexicographic_ordering(): - """ - Although one might like 2.7.10 to be greater than 2.7.3, - the marker spec only supports lexicographic ordering. - """ - assert evaluate_marker("python_full_version > '2.7.3'") is False +@mock.patch('platform.python_version', return_value='2.7.10') +def test_ordering(python_version_mock): + assert evaluate_marker("python_full_version > '2.7.3'") is True