Merge pull request #845 from nolar/pypy-in-ci

Support PyPy 3.7 officially and auto-test it in CI

.github/workflows/ci.yaml

@@ -69,6 +69,33 @@ jobs:
           env_vars: PYTHON
         continue-on-error: true
 
+  # Only the core functionality is tested: no e2e or functional tests (for simplicity).
+  # No coverage: PyPy performs extremely poorly with tracing/coverage (13 mins vs. 3 mins).
+  # Extra time: 2-3 mins for building the dependencies (since no binary wheels are available).
+  # Extra time: PyPy is good with JIT for repetitive code; tests are too unique for JIT.
+  pypy-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        install-extras: [ "", "full-auth" ]
+        python-version: [ "pypy-3.7" ]
+    name: Python ${{ matrix.python-version }} ${{ matrix.install-extras }}
+    runs-on: ubuntu-20.04
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - run: sudo apt-get update && sudo apt-get install libxml2-dev libxslt-dev
+      - run: pip install wheel
+
+      - run: pip install -r requirements.txt
+      - run: pip install -e .[${{ matrix.install-extras }}]
+        if: ${{ matrix.install-extras }}
+      - run: pytest --color=yes --no-cov
+
   functional:
     strategy:
       fail-fast: false

.github/workflows/thorough.yaml

@@ -71,6 +71,33 @@ jobs:
           flags: unit
           env_vars: PYTHON
 
+  # Only the core functionality is tested: no e2e or functional tests (for simplicity).
+  # No coverage: PyPy performs extremely poorly with tracing/coverage (13 mins vs. 3 mins).
+  # Extra time: 2-3 mins for building the dependencies (since no binary wheels are available).
+  # Extra time: PyPy is good with JIT for repetitive code; tests are too unique for JIT.
+  pypy-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        install-extras: [ "", "full-auth" ]
+        python-version: [ "pypy-3.7" ]
+    name: Python ${{ matrix.python-version }} ${{ matrix.install-extras }}
+    runs-on: ubuntu-20.04
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - run: sudo apt-get update && sudo apt-get install libxml2-dev libxslt-dev
+      - run: pip install wheel
+
+      - run: pip install -r requirements.txt
+      - run: pip install -e .[${{ matrix.install-extras }}]
+        if: ${{ matrix.install-extras }}
+      - run: pytest --color=yes --no-cov
+
   functional:
     strategy:
       fail-fast: false

README.md

@@ -124,6 +124,10 @@ That easy! For more features, see the [documentation](https://kopf.readthedocs.i
 
 ## Usage
 
+Python 3.7+ is required:
+[CPython](https://www.python.org/) and [PyPy](https://www.pypy.org/)
+are officially supported and tested; other Python implementations can work too.
+
 We assume that when the operator is executed in the cluster, it must be packaged
 into a docker image with a CI/CD tool of your preference.
 

docs/install.rst

@@ -4,6 +4,10 @@ Installation
 
 .. highlight:: bash
 
+Prerequisites:
+
+* Python >= 3.7 (CPython and PyPy are officially tested and supported).
+
 To install Kopf::
 
     pip install kopf

kopf/_kits/webhooks.py

@@ -160,9 +160,11 @@ async def _serve_fn(request: aiohttp.web.Request) -> aiohttp.web.Response:
         runner = aiohttp.web.AppRunner(app, handle_signals=False)
         await runner.setup()
         try:
+            # Note: reuse_port is mostly (but not only) for fast-running tests with SSL sockets;
+            # multi-threaded sockets are not really used -- high load is not expected for webhooks.
             addr = self.addr or None  # None is aiohttp's "any interface"
             port = self.port or self._allocate_free_port()
-            site = aiohttp.web.TCPSite(runner, addr, port, ssl_context=context)
+            site = aiohttp.web.TCPSite(runner, addr, port, ssl_context=context, reuse_port=True)
             await site.start()
 
             # Log with the actual URL: normalised, with hostname/port set.

requirements.txt

@@ -14,7 +14,8 @@ freezegun
 import-linter
 isort>=5.5.0
 lxml
-mypy==0.910
+# Mypy requires typed-ast, which is broken on PyPy 3.7 (could work in PyPy 3.8).
+mypy==0.910; implementation_name == "cpython"
 pre-commit
 pyngrok
 pytest>=6.0.0

tests/reactor/test_queueing.py

@@ -16,6 +16,7 @@
 """
 import asyncio
 import contextlib
+import gc
 import weakref
 
 import async_timeout
@@ -224,6 +225,10 @@ async def test_garbage_collection_of_streams(settings, stream, events, unique, w
     # The jobs can take a tiny moment more, but this is noticeable in the tests.
     await asyncio.sleep(0.1)
 
+    # For PyPy: force the gc! (GC can be delayed in PyPy, unlike in CPython.)
+    # https://doc.pypy.org/en/latest/cpython_differences.html#differences-related-to-garbage-collection-strategies
+    gc.collect()
+
     # Truly garbage-collected? Memory freed?
     assert all([ref() is None for ref in refs])