From 64b4318bce1a2ba8dcc0b8f7cbaefeb029c12419 Mon Sep 17 00:00:00 2001 From: Matthieu Darbois Date: Mon, 27 Mar 2023 23:37:16 +0200 Subject: [PATCH] Win: fix running tests in a virtual environment (#2216) On windows, starting with python 3.7, virtual environments use a venvlauncher startup process This does not play well when counting spawned processes or when relying on the pid of the spawned process to do some checks e.g. connection check per pid This commit detects this situation and uses the base python executable to spawn processes when required. Signed-off-by: mayeut --- .github/workflows/build.yml | 60 +++++++--------------------------- psutil/tests/__init__.py | 35 ++++++++++++-------- psutil/tests/test_misc.py | 2 ++ psutil/tests/test_process.py | 7 ++-- psutil/tests/test_testutils.py | 4 ++- pyproject.toml | 6 ---- 6 files changed, 42 insertions(+), 72 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e2d015c1d..eb6996daa 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,14 +22,21 @@ name: build jobs: # Linux + macOS + Windows Python 3 py3: - name: py3-${{ matrix.os }} + name: py3-${{ matrix.os }}-${{ startsWith(matrix.os, 'windows') && matrix.archs || 'all' }} runs-on: ${{ matrix.os }} timeout-minutes: 20 strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-12, windows-2019] - # python: ["3.6", "3.11"] + include: + - os: ubuntu-latest + archs: "x86_64 i686" + - os: macos-12 + archs: "x86_64 arm64" + - os: windows-2019 + archs: "AMD64" + - os: windows-2019 + archs: "x86" steps: - name: Cancel previous runs @@ -44,6 +51,8 @@ jobs: - name: Create wheels + run tests uses: pypa/cibuildwheel@v2.11.2 + env: + CIBW_ARCHS: "${{ matrix.archs }}" - name: Upload wheels uses: actions/upload-artifact@v3 @@ -58,51 +67,6 @@ jobs: python setup.py sdist mv dist/psutil*.tar.gz wheelhouse/ - # Windows cp37+ tests - # psutil tests do not like running from a virtualenv with python>=3.7 so - # not using cibuildwheel for those. run them "manually" with this job. - py3-windows-tests: - name: py3-windows-test-${{ matrix.python }}-${{ matrix.architecture }} - needs: py3 - runs-on: windows-2019 - timeout-minutes: 20 - strategy: - fail-fast: false - matrix: - python: ["3.7", "3.8", "3.9", "3.10", "3.11"] - architecture: ["x86", "x64"] - - steps: - - name: Cancel previous runs - uses: styfle/cancel-workflow-action@0.9.1 - with: - access_token: ${{ github.token }} - - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "${{ matrix.python }}" - architecture: "${{ matrix.architecture }}" - - - name: Download wheels - uses: actions/download-artifact@v3 - with: - name: wheels - path: wheelhouse - - - name: Run tests - run: | - mkdir .tests - cd .tests - pip install $(find ../wheelhouse -name '*-cp36-abi3-${{ matrix.architecture == 'x86' && 'win32' || 'win_amd64'}}.whl')[test] - export PYTHONWARNINGS=always - export PYTHONUNBUFFERED=1 - export PSUTIL_DEBUG=1 - python ../psutil/tests/runner.py - python ../psutil/tests/test_memleaks.py - shell: bash - # Linux + macOS + Python 2 py2: name: py2-${{ matrix.os }} diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 61a06e216..e3766eea1 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -41,7 +41,6 @@ import psutil from psutil import AIX -from psutil import FREEBSD from psutil import LINUX from psutil import MACOS from psutil import POSIX @@ -80,8 +79,8 @@ __all__ = [ # constants 'APPVEYOR', 'DEVNULL', 'GLOBAL_TIMEOUT', 'TOLERANCE_SYS_MEM', 'NO_RETRIES', - 'PYPY', 'PYTHON_EXE', 'ROOT_DIR', 'SCRIPTS_DIR', 'TESTFN_PREFIX', - 'UNICODE_SUFFIX', 'INVALID_UNICODE_SUFFIX', + 'PYPY', 'PYTHON_EXE', 'PYTHON_EXE_ENV', 'ROOT_DIR', 'SCRIPTS_DIR', + 'TESTFN_PREFIX', 'UNICODE_SUFFIX', 'INVALID_UNICODE_SUFFIX', 'CI_TESTING', 'VALID_PROC_STATUSES', 'TOLERANCE_DISK_USAGE', 'IS_64BIT', "HAS_CPU_AFFINITY", "HAS_CPU_FREQ", "HAS_ENVIRON", "HAS_PROC_IO_COUNTERS", "HAS_IONICE", "HAS_MEMORY_MAPS", "HAS_PROC_CPU_NUM", "HAS_RLIMIT", @@ -240,13 +239,21 @@ def attempt(exe): else: return exe - if GITHUB_ACTIONS: - if PYPY: - return which("pypy3") if PY3 else which("pypy") - elif FREEBSD: - return os.path.realpath(sys.executable) - else: - return which('python') + env = os.environ.copy() + + # On Windows, starting with python 3.7, virtual environments use a + # venv launcher startup process. This does not play well when + # counting spawned processes, or when relying on the PID of the + # spawned process to do some checks, e.g. connections check per PID. + # Let's use the base python in this case. + base = getattr(sys, "_base_executable", None) + if WINDOWS and sys.version_info >= (3, 7) and base is not None: + # We need to set __PYVENV_LAUNCHER__ to sys.executable for the + # base python executable to know about the environment. + env["__PYVENV_LAUNCHER__"] = sys.executable + return base, env + elif GITHUB_ACTIONS: + return sys.executable, env elif MACOS: exe = \ attempt(sys.executable) or \ @@ -255,14 +262,14 @@ def attempt(exe): attempt(psutil.Process().exe()) if not exe: raise ValueError("can't find python exe real abspath") - return exe + return exe, env else: exe = os.path.realpath(sys.executable) assert os.path.exists(exe), exe - return exe + return exe, env -PYTHON_EXE = _get_py_exe() +PYTHON_EXE, PYTHON_EXE_ENV = _get_py_exe() DEVNULL = open(os.devnull, 'r+') atexit.register(DEVNULL.close) @@ -351,7 +358,7 @@ def spawn_testproc(cmd=None, **kwds): kwds.setdefault("stdin", DEVNULL) kwds.setdefault("stdout", DEVNULL) kwds.setdefault("cwd", os.getcwd()) - kwds.setdefault("env", os.environ) + kwds.setdefault("env", PYTHON_EXE_ENV) if WINDOWS: # Prevents the subprocess to open error dialogs. This will also # cause stderr to be suppressed, which is suboptimal in order diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 44922216a..afa60b1c3 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -45,6 +45,7 @@ from psutil.tests import HAS_SENSORS_FANS from psutil.tests import HAS_SENSORS_TEMPERATURES from psutil.tests import PYTHON_EXE +from psutil.tests import PYTHON_EXE_ENV from psutil.tests import SCRIPTS_DIR from psutil.tests import PsutilTestCase from psutil.tests import mock @@ -820,6 +821,7 @@ class TestScripts(PsutilTestCase): @staticmethod def assert_stdout(exe, *args, **kwargs): + kwargs.setdefault("env", PYTHON_EXE_ENV) exe = '%s' % os.path.join(SCRIPTS_DIR, exe) cmd = [PYTHON_EXE, exe] for arg in args: diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 26869e983..df6a84c39 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -52,6 +52,7 @@ from psutil.tests import MACOS_11PLUS from psutil.tests import PYPY from psutil.tests import PYTHON_EXE +from psutil.tests import PYTHON_EXE_ENV from psutil.tests import PsutilTestCase from psutil.tests import ThreadTask from psutil.tests import call_until @@ -1543,7 +1544,7 @@ def test_misc(self): # Not sure what to do though. cmd = [PYTHON_EXE, "-c", "import time; time.sleep(60);"] with psutil.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) as proc: + stderr=subprocess.PIPE, env=PYTHON_EXE_ENV) as proc: proc.name() proc.cpu_times() proc.stdin @@ -1559,7 +1560,7 @@ def test_ctx_manager(self): with psutil.Popen([PYTHON_EXE, "-V"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, - stdin=subprocess.PIPE) as proc: + stdin=subprocess.PIPE, env=PYTHON_EXE_ENV) as proc: proc.communicate() assert proc.stdout.closed assert proc.stderr.closed @@ -1572,7 +1573,7 @@ def test_kill_terminate(self): # diverges from that. cmd = [PYTHON_EXE, "-c", "import time; time.sleep(60);"] with psutil.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) as proc: + stderr=subprocess.PIPE, env=PYTHON_EXE_ENV) as proc: proc.terminate() proc.wait() self.assertRaises(psutil.NoSuchProcess, proc.terminate) diff --git a/psutil/tests/test_testutils.py b/psutil/tests/test_testutils.py index 0298ea4e3..e757e017a 100755 --- a/psutil/tests/test_testutils.py +++ b/psutil/tests/test_testutils.py @@ -30,6 +30,7 @@ from psutil.tests import COVERAGE from psutil.tests import HAS_CONNECTIONS_UNIX from psutil.tests import PYTHON_EXE +from psutil.tests import PYTHON_EXE_ENV from psutil.tests import PsutilTestCase from psutil.tests import TestMemoryLeak from psutil.tests import bind_socket @@ -260,7 +261,8 @@ def test_terminate(self): terminate(p) # by psutil.Popen cmd = [PYTHON_EXE, "-c", "import time; time.sleep(60);"] - p = psutil.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p = psutil.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + env=PYTHON_EXE_ENV) terminate(p) self.assertProcessGone(p) terminate(p) diff --git a/pyproject.toml b/pyproject.toml index 435cc989c..c8cb62af0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,9 +48,3 @@ test-command = [ [tool.cibuildwheel.macos] archs = ["x86_64", "arm64"] - -[tool.cibuildwheel.windows] -# psutil tests do not like running from a virtualenv with python>=3.7 -# restrict build & tests to cp36 -# cp36-abi3 wheels will need to be tested outside cibuildwheel for python>=3.7 -build = "cp36-*"