From 405f2febc585d17c76f0be990fb76c1d68ff23b4 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Thu, 8 Dec 2022 17:15:22 +0000 Subject: [PATCH 1/9] First attempt at reworking for tox 4 --- tox_pypi_filter/plugin.py | 60 +++++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/tox_pypi_filter/plugin.py b/tox_pypi_filter/plugin.py index 41b9464..66fcdb4 100644 --- a/tox_pypi_filter/plugin.py +++ b/tox_pypi_filter/plugin.py @@ -8,7 +8,11 @@ import urllib.request from textwrap import indent -import pluggy +from tox.plugin import impl +from tox.config.cli.parser import ToxParser +from tox.config.sets import ConfigSet, EnvConfigSet +from tox.session.state import State + from pkg_resources import DistributionNotFound, get_distribution try: @@ -16,7 +20,6 @@ except DistributionNotFound: pass -hookimpl = pluggy.HookimplMarker("tox") HELP = ("Specify version constraints for packages which are then applied by " "setting up a proxy PyPI server. If giving multiple constraints, you " @@ -25,24 +28,31 @@ "http://, https://, or ftp://).") -@hookimpl -def tox_addoption(parser): - parser.add_argument('--pypi-filter', dest='pypi_filter', help=HELP) - parser.add_testenv_attribute('pypi_filter', 'string', help=HELP) +@impl +def tox_add_option(parser: ToxParser) -> None: + parser.add_argument('--pypi-filter', action="store", type=str, of_type=str) + + +@impl +def tox_add_core_config(core_conf: ConfigSet, state: State) -> None: + core_conf.add_config('pypi_filter', 'string', default="", desc=HELP) SERVER_PROCESS = {} -@hookimpl -def tox_testenv_create(venv, action): +@impl +def tox_add_env_config(env_conf: EnvConfigSet, state: State) -> None: # Skip the environment used for creating the tarball - if venv.name == ".package": + if env_conf.name == ".pkg": return global SERVER_PROCESS - pypi_filter = venv.envconfig.config.option.pypi_filter or venv.envconfig.pypi_filter + pypi_filter_config = state.conf.core["pypi_filter"] + pypi_filter_cli = state.conf.options.pypi_filter + + pypi_filter = pypi_filter_cli or pypi_filter_config if not pypi_filter: return @@ -72,33 +82,27 @@ def tox_testenv_create(venv, action): sock.close() # Run pypicky - print(f"{venv.name}: Starting tox-pypi-filter server with the following requirements:") + print(f"{env_conf.name}: Starting tox-pypi-filter server with the following requirements:") print(indent(contents.strip(), ' ')) - SERVER_PROCESS[venv.name] = subprocess.Popen([sys.executable, '-m', 'pypicky', - reqfile, '--port', str(port), '--quiet']) + SERVER_PROCESS[env_conf.name] = subprocess.Popen([sys.executable, '-m', 'pypicky', + reqfile, '--port', str(port), '--quiet']) # FIXME: properly check that the server has started up time.sleep(2) - venv.envconfig.config.indexserver['default'].url = f'http://localhost:{port}' + env_config = env_conf.load("setenv") + if "PIP_INDEX_URL" in env_config: + raise ValueError("Can not use tox-pypi-filter if already setting the PIP_INDEX_URL env var.") + env_config.update({"PIP_INDEX_URL": f'http://localhost:{port}'}) -@hookimpl -def tox_runtest_post(venv): +@impl +def tox_after_run_commands(tox_env, exit_code, outcomes): + print("After run commands") global SERVER_PROCESS - proc = SERVER_PROCESS.pop(venv.name, None) + proc = SERVER_PROCESS.pop(tox_env.name, None) if proc: - print(f"{venv.name}: Shutting down tox-pypi-filter server") + print(f"{tox_env.name}: Shutting down tox-pypi-filter server") proc.terminate() - - -@hookimpl -def tox_cleanup(session): - global SERVER_PROCESS - - for venv, process in SERVER_PROCESS.items(): - print(f"{venv}: Shutting down tox-pypi-filter server.") - process.terminate() - SERVER_PROCESS = {} From 0dd9d79b4465b647ce12ac4152278db38092a598 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Fri, 9 Dec 2022 16:52:52 +0000 Subject: [PATCH 2/9] Only tox 4 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 1118825..fedeea3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,7 +11,7 @@ zip_safe = True packages = find: setup_requires = setuptools_scm install_requires = - tox<4 + tox>=4 setuptools pypicky>=0.5 From 530ff646d82df1e339e30cf8e239d9d7c193847b Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Mon, 12 Dec 2022 14:31:19 +0000 Subject: [PATCH 3/9] Fix config location --- tox_pypi_filter/plugin.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/tox_pypi_filter/plugin.py b/tox_pypi_filter/plugin.py index 66fcdb4..353f223 100644 --- a/tox_pypi_filter/plugin.py +++ b/tox_pypi_filter/plugin.py @@ -10,7 +10,7 @@ from tox.plugin import impl from tox.config.cli.parser import ToxParser -from tox.config.sets import ConfigSet, EnvConfigSet +from tox.config.sets import EnvConfigSet from tox.session.state import State from pkg_resources import DistributionNotFound, get_distribution @@ -33,25 +33,21 @@ def tox_add_option(parser: ToxParser) -> None: parser.add_argument('--pypi-filter', action="store", type=str, of_type=str) -@impl -def tox_add_core_config(core_conf: ConfigSet, state: State) -> None: - core_conf.add_config('pypi_filter', 'string', default="", desc=HELP) - - SERVER_PROCESS = {} @impl def tox_add_env_config(env_conf: EnvConfigSet, state: State) -> None: + env_conf.add_config('pypi_filter', default=None, desc=HELP, of_type=str) + # Skip the environment used for creating the tarball if env_conf.name == ".pkg": return global SERVER_PROCESS - pypi_filter_config = state.conf.core["pypi_filter"] + pypi_filter_config = env_conf.load("pypi_filter") pypi_filter_cli = state.conf.options.pypi_filter - pypi_filter = pypi_filter_cli or pypi_filter_config if not pypi_filter: @@ -99,7 +95,6 @@ def tox_add_env_config(env_conf: EnvConfigSet, state: State) -> None: @impl def tox_after_run_commands(tox_env, exit_code, outcomes): - print("After run commands") global SERVER_PROCESS proc = SERVER_PROCESS.pop(tox_env.name, None) From e215f4f1ee0aed327f09286b28f557abd57cf942 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Wed, 8 Mar 2023 14:41:53 +0000 Subject: [PATCH 4/9] Change to pyproject only config --- pyproject.toml | 25 +++++++++++++++++++++++-- setup.cfg | 29 ----------------------------- setup.py | 2 -- 3 files changed, 23 insertions(+), 33 deletions(-) delete mode 100644 setup.cfg delete mode 100644 setup.py diff --git a/pyproject.toml b/pyproject.toml index 2d6e225..3cce1dd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,24 @@ [build-system] -requires = ["setuptools", "wheel"] -build-backend = 'setuptools.build_meta' +requires = ["setuptools", "setuptools-scm"] +build-backend = "setuptools.build_meta" + +[project] +name = "tox-pypi-filter" +authors = [ + {name = "Thomas Robitaille", email = "thomas.robitaille@gmail.com"}, + {name = "Stuart Mumford", email = "stuart@cadair.com"}, +] +description = "Implement a --pypi-filter option for tox" +readme = "README.rst" +requires-python = ">=3.8" +license = {text = "BSD-2-Clause"} +dependencies = [ + "tox>=4.0.9", + "pypicky>=0.5" +] +dynamic = ["version"] + +[project.entry-points."tox"] +pypi-filter = "tox_pypi_filter.plugin" + +[tool.setuptools_scm] diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index fedeea3..0000000 --- a/setup.cfg +++ /dev/null @@ -1,29 +0,0 @@ -[metadata] -name = tox-pypi-filter -author = Thomas Robitaille -author_email = thomas.robitaille@gmail.com -description = Implement a --pypi-filter option for tox -long_description = file: README.rst -url = https://github.com/astrofrog/tox-pypi-filter - -[options] -zip_safe = True -packages = find: -setup_requires = setuptools_scm -install_requires = - tox>=4 - setuptools - pypicky>=0.5 - -[options.entry_points] -tox = - pypi-filter = tox_pypi_filter.plugin - -[isort] -length_sort_stdlib = True - -[flake8] -max-line-length = 100 - -[pycodestyle] -max_line_length = 100 diff --git a/setup.py b/setup.py deleted file mode 100644 index 3f43000..0000000 --- a/setup.py +++ /dev/null @@ -1,2 +0,0 @@ -from setuptools import setup -setup(use_scm_version=True,) From 071f06de3c1ff3c21b929f91ab0121e354077848 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Wed, 8 Mar 2023 15:31:01 +0000 Subject: [PATCH 5/9] Use new tox hooks --- tox_pypi_filter/plugin.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tox_pypi_filter/plugin.py b/tox_pypi_filter/plugin.py index 353f223..73d9bc2 100644 --- a/tox_pypi_filter/plugin.py +++ b/tox_pypi_filter/plugin.py @@ -7,8 +7,10 @@ import urllib.parse import urllib.request from textwrap import indent +from typing import Any from tox.plugin import impl +from tox.tox_env.api import ToxEnv from tox.config.cli.parser import ToxParser from tox.config.sets import EnvConfigSet from tox.session.state import State @@ -35,19 +37,17 @@ def tox_add_option(parser: ToxParser) -> None: SERVER_PROCESS = {} - @impl def tox_add_env_config(env_conf: EnvConfigSet, state: State) -> None: env_conf.add_config('pypi_filter', default=None, desc=HELP, of_type=str) - # Skip the environment used for creating the tarball - if env_conf.name == ".pkg": - return +@impl +def tox_on_install(tox_env: ToxEnv, arguments: Any, section: str, of_type: str) -> None: global SERVER_PROCESS - pypi_filter_config = env_conf.load("pypi_filter") - pypi_filter_cli = state.conf.options.pypi_filter + pypi_filter_config = tox_env.conf.load("pypi_filter") + pypi_filter_cli = tox_env.options.pypi_filter pypi_filter = pypi_filter_cli or pypi_filter_config if not pypi_filter: @@ -78,23 +78,23 @@ def tox_add_env_config(env_conf: EnvConfigSet, state: State) -> None: sock.close() # Run pypicky - print(f"{env_conf.name}: Starting tox-pypi-filter server with the following requirements:") + print(f"{tox_env.name}: Starting tox-pypi-filter server with the following requirements:") print(indent(contents.strip(), ' ')) - SERVER_PROCESS[env_conf.name] = subprocess.Popen([sys.executable, '-m', 'pypicky', + SERVER_PROCESS[tox_env.name] = subprocess.Popen([sys.executable, '-m', 'pypicky', reqfile, '--port', str(port), '--quiet']) # FIXME: properly check that the server has started up time.sleep(2) - env_config = env_conf.load("setenv") + env_config = tox_env.conf.load("setenv") if "PIP_INDEX_URL" in env_config: raise ValueError("Can not use tox-pypi-filter if already setting the PIP_INDEX_URL env var.") env_config.update({"PIP_INDEX_URL": f'http://localhost:{port}'}) @impl -def tox_after_run_commands(tox_env, exit_code, outcomes): +def tox_env_teardown(tox_env): global SERVER_PROCESS proc = SERVER_PROCESS.pop(tox_env.name, None) From 6a35dd3f2011113541691dd13c11e42946b0fda3 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Wed, 8 Mar 2023 15:41:41 +0000 Subject: [PATCH 6/9] Ignore comments in the requirements file --- tox_pypi_filter/plugin.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tox_pypi_filter/plugin.py b/tox_pypi_filter/plugin.py index 73d9bc2..d5dcbda 100644 --- a/tox_pypi_filter/plugin.py +++ b/tox_pypi_filter/plugin.py @@ -67,9 +67,11 @@ def tox_on_install(tox_env: ToxEnv, arguments: Any, section: str, of_type: str) # If we get a blank set of requirements then we don't do anything. with open(reqfile, "r") as fobj: - contents = fobj.read() + contents = fobj.readlines() + contents = list(filter(lambda line: not line.startswith("#"), contents)) if not contents: return + contents = "\n".join(contents) # Find available port sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -82,7 +84,7 @@ def tox_on_install(tox_env: ToxEnv, arguments: Any, section: str, of_type: str) print(indent(contents.strip(), ' ')) SERVER_PROCESS[tox_env.name] = subprocess.Popen([sys.executable, '-m', 'pypicky', - reqfile, '--port', str(port), '--quiet']) + reqfile, '--port', str(port), '--quiet']) # FIXME: properly check that the server has started up time.sleep(2) From 3d78c17028cbce829c9c0082a1f14f4ea73e0a46 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Wed, 8 Mar 2023 16:27:40 +0000 Subject: [PATCH 7/9] Adapt for the fact the hook gets called multiple times --- tox_pypi_filter/plugin.py | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/tox_pypi_filter/plugin.py b/tox_pypi_filter/plugin.py index d5dcbda..48a3dcf 100644 --- a/tox_pypi_filter/plugin.py +++ b/tox_pypi_filter/plugin.py @@ -36,6 +36,7 @@ def tox_add_option(parser: ToxParser) -> None: SERVER_PROCESS = {} +SERVER_URLS = {} @impl def tox_add_env_config(env_conf: EnvConfigSet, state: State) -> None: @@ -44,7 +45,11 @@ def tox_add_env_config(env_conf: EnvConfigSet, state: State) -> None: @impl def tox_on_install(tox_env: ToxEnv, arguments: Any, section: str, of_type: str) -> None: - global SERVER_PROCESS + # Do not set the environment variable for the custom index if we are in the .pkg step + if tox_env.name == ".pkg": + return + + global SERVER_PROCESS, SERVER_URLS pypi_filter_config = tox_env.conf.load("pypi_filter") pypi_filter_cli = tox_env.options.pypi_filter @@ -73,26 +78,31 @@ def tox_on_install(tox_env: ToxEnv, arguments: Any, section: str, of_type: str) return contents = "\n".join(contents) - # Find available port - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.bind(('localhost', 0)) - port = sock.getsockname()[1] - sock.close() + # We might have already setup the process for this env at an earlier call of this function. + if tox_env.name not in SERVER_PROCESS: + # Find available port + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.bind(('localhost', 0)) + port = sock.getsockname()[1] + sock.close() - # Run pypicky - print(f"{tox_env.name}: Starting tox-pypi-filter server with the following requirements:") - print(indent(contents.strip(), ' ')) + # Run pypicky + print(f"{tox_env.name}: Starting tox-pypi-filter server with the following requirements:") + print(indent(contents.strip(), ' ')) - SERVER_PROCESS[tox_env.name] = subprocess.Popen([sys.executable, '-m', 'pypicky', - reqfile, '--port', str(port), '--quiet']) + SERVER_URLS[tox_env.name] = f'http://localhost:{port}' + SERVER_PROCESS[tox_env.name] = subprocess.Popen([sys.executable, '-m', 'pypicky', + reqfile, '--port', str(port), '--quiet']) # FIXME: properly check that the server has started up time.sleep(2) - env_config = tox_env.conf.load("setenv") - if "PIP_INDEX_URL" in env_config: + # If PIP_INDEX_URL is configured in config it will conflict + if "PIP_INDEX_URL" in tox_env.conf.load("setenv"): raise ValueError("Can not use tox-pypi-filter if already setting the PIP_INDEX_URL env var.") - env_config.update({"PIP_INDEX_URL": f'http://localhost:{port}'}) + + # Add the index url to the env vars just for this install (as oppsed to the global config) + tox_env.environment_variables["PIP_INDEX_URL"] = SERVER_URLS[tox_env.name] @impl From f8cfaf99cb24cdde8c48309d63cead71d36f7c8f Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Wed, 8 Mar 2023 17:49:24 +0000 Subject: [PATCH 8/9] unpin tox --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1a1589e..0e0da22 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,6 @@ jobs: uses: OpenAstronomy/github-actions-workflows/.github/workflows/tox.yml@v1 with: submodules: false - toxdeps: "'tox<4'" envs: | - linux: codestyle - windows: py38-test-cli From c3d00d3d76b9c4ea53852c5f680f67e28b80c1e9 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Wed, 8 Mar 2023 17:52:30 +0000 Subject: [PATCH 9/9] lint --- tox_pypi_filter/plugin.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tox_pypi_filter/plugin.py b/tox_pypi_filter/plugin.py index 48a3dcf..0b3cc03 100644 --- a/tox_pypi_filter/plugin.py +++ b/tox_pypi_filter/plugin.py @@ -38,6 +38,7 @@ def tox_add_option(parser: ToxParser) -> None: SERVER_PROCESS = {} SERVER_URLS = {} + @impl def tox_add_env_config(env_conf: EnvConfigSet, state: State) -> None: env_conf.add_config('pypi_filter', default=None, desc=HELP, of_type=str)