From c279d3c392972bfef942e7a451d0e0d425b887d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20H=C3=B6sel?= Date: Wed, 5 Jun 2024 19:30:58 +0200 Subject: [PATCH] lock transitive python dependencies with pip-tools After yet another breaking package update (https://github.com/docker/docker-py/pull/3257), its high time we permanently pin all development packages *and their dependencies* to a known.-good version. pip-tools lets us do that without losing the simple requirements.txt file that allows easy installation, so lets use that. --- .github/renovate.json5 | 11 +- CONTRIBUTING.md | 10 + requirements.in | 36 ++++ requirements.txt | 375 +++++++++++++++++++++++++++++++++-- scripts/setup.sh | 4 +- tests/roles/requirements.txt | 4 +- 6 files changed, 414 insertions(+), 26 deletions(-) create mode 100644 requirements.in mode change 100644 => 120000 tests/roles/requirements.txt diff --git a/.github/renovate.json5 b/.github/renovate.json5 index 4574387d..6d4db2d7 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -1,5 +1,14 @@ { "extends": [ "github>maxhoesel-ansible/.github:renovate-config" - ] + ], + "pip-compile": { + "fileMatch": ["^requirements\\.txt$"] + }, + "pip_requirements": { + "enabled": false + }, + "pip_setup": { + "enabled": false + } } diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 063ed3f2..f24fbc6a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -170,6 +170,16 @@ The CI also builds the docs to ensure they don't break silently. ## Maintainer information +### Updating Dependencies + +While the *Ansible* collection itself doesn't have any dependencies outside of ansible itself, the tooling used to build and test the collection does. +We use [`pip-tools`](https://github.com/jazzband/pip-tools/) to lock these dependencies to a specific version for testing. +This prevents random CI failures because of [`requests` updates et. al.](https://github.com/docker/docker-py/pull/3257), but still gives us a simple `requirements.txt` that anyone can install. + +The direct dependencies are stored in `requirements.in`, use `pip-compile requirements.in` to generate a new `requirements.txt`. +You **must** add the `requirements.in` file, else renovate [won't be able to resolve and update dependencies in CI!](https://docs.renovatebot.com/modules/manager/pip-compile/#assumption-of-header-with-a-command) + + ### Raising minimum supported step versions 1. Change the versions in [`plugins/module_utils/constants.py`](./plugins/module_utils/constants.py) diff --git a/requirements.in b/requirements.in new file mode 100644 index 00000000..d8c41972 --- /dev/null +++ b/requirements.in @@ -0,0 +1,36 @@ +# This file is only a template! +# Use requirements.txt generated by pip-compile to install dependencies + +# Requirements for developing this collection +# Includes utilities, CLI helpers and so on + +# Linting & Formatting +ansible-lint==24.5.0 +pylint==3.2.1 +autopep8==2.1.0 +pre-commit==3.7.1 + +# Testing libraries +pytest==8.2.0 +pytest-virtualenv==1.7.0 +docker==7.1.0 +# Dependencies for executing the role scenarios. +molecule==6.0.2 +molecule-plugins[docker]==23.4.1 + +# Utility packages used in scripts +pyyaml==6.0.1 +packaging==24.0 +# Generating requirements and syncing venv +pip-tools==7.4.1 + +# Also include a version of ansible-core for IDE hints +# and as the default version used in tests. +# It is also needed for docs generation +ansible-core==2.16.6 + +# Docs +antsibull-docs==2.11.0 +ansible-pygments==0.1.1 +sphinx==7.3.7 +sphinx-ansible-theme==0.10.3 diff --git a/requirements.txt b/requirements.txt index 6cbc591a..2ab8f4d9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,28 +1,363 @@ -# Requirements for developing this collection -# Includes utilities, CLI helpers and so on - -# Linting & Formatting +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile requirements.in +# +aiofiles==23.2.1 + # via antsibull-core +aiohttp==3.9.5 + # via + # antsibull-core + # antsibull-docs +aiosignal==1.3.1 + # via aiohttp +alabaster==0.7.16 + # via sphinx +annotated-types==0.7.0 + # via pydantic +ansible-compat==24.6.1 + # via + # ansible-lint + # molecule +ansible-core==2.16.6 + # via + # -r requirements.in + # ansible-compat + # ansible-lint + # molecule ansible-lint==24.5.0 -pylint==3.2.1 + # via -r requirements.in +ansible-pygments==0.1.1 + # via + # -r requirements.in + # antsibull-docs + # sphinx-ansible-theme +antsibull-changelog==0.28.0 + # via antsibull-docs +antsibull-core==3.0.1 + # via antsibull-docs +antsibull-docs==2.11.0 + # via -r requirements.in +antsibull-docs-parser==1.0.1 + # via antsibull-docs +astroid==3.2.2 + # via pylint +asyncio-pool==0.6.0 + # via antsibull-docs +attrs==23.2.0 + # via + # aiohttp + # jsonschema + # referencing autopep8==2.1.0 + # via -r requirements.in +babel==2.15.0 + # via sphinx +black==24.4.2 + # via ansible-lint +bracex==2.4 + # via wcmatch +build==1.2.1 + # via + # antsibull-core + # pip-tools +certifi==2024.6.2 + # via requests +cffi==1.16.0 + # via cryptography +cfgv==3.4.0 + # via pre-commit +charset-normalizer==3.3.2 + # via requests +click==8.1.7 + # via + # black + # click-help-colors + # molecule + # pip-tools + # typer +click-help-colors==0.9.4 + # via molecule +contextlib2==21.6.0 + # via pytest-shutil +cryptography==42.0.8 + # via ansible-core +dill==0.3.8 + # via pylint +distlib==0.3.8 + # via virtualenv +distro==1.9.0 + # via selinux +docker==7.1.0 + # via + # -r requirements.in + # molecule-plugins +docutils==0.20.1 + # via + # antsibull-changelog + # antsibull-docs + # rstcheck-core + # sphinx + # sphinx-rtd-theme +enrich==1.2.7 + # via molecule +execnet==2.1.1 + # via pytest-shutil +filelock==3.14.0 + # via + # ansible-lint + # virtualenv +frozenlist==1.4.1 + # via + # aiohttp + # aiosignal +identify==2.5.36 + # via pre-commit +idna==3.7 + # via + # requests + # yarl +imagesize==1.4.1 + # via sphinx +importlib-metadata==7.1.0 + # via ansible-lint +iniconfig==2.0.0 + # via pytest +isort==5.13.2 + # via pylint +jinja2==3.1.4 + # via + # ansible-core + # antsibull-docs + # molecule + # sphinx +jsonschema==4.22.0 + # via + # ansible-compat + # ansible-lint + # molecule +jsonschema-specifications==2023.12.1 + # via jsonschema +markdown-it-py==3.0.0 + # via rich +markupsafe==2.1.5 + # via jinja2 +mccabe==0.7.0 + # via pylint +mdurl==0.1.2 + # via markdown-it-py +mock==5.1.0 + # via pytest-shutil +molecule==6.0.2 + # via + # -r requirements.in + # molecule-plugins +molecule-plugins[docker]==23.4.1 + # via -r requirements.in +multidict==6.0.5 + # via + # aiohttp + # yarl +mypy-extensions==1.0.0 + # via black +nodeenv==1.9.1 + # via pre-commit +packaging==24.0 + # via + # -r requirements.in + # ansible-compat + # ansible-core + # ansible-lint + # antsibull-changelog + # antsibull-core + # antsibull-docs + # black + # build + # molecule + # pytest + # sphinx +path==16.14.0 + # via path-py +path-py==12.5.0 + # via pytest-shutil +pathspec==0.12.1 + # via + # ansible-lint + # black + # yamllint +perky==0.9.2 + # via antsibull-core +pip-tools==7.4.1 + # via -r requirements.in +platformdirs==4.2.2 + # via + # black + # pylint + # virtualenv +pluggy==1.5.0 + # via + # molecule + # pytest pre-commit==3.7.1 - -# Testing libraries + # via -r requirements.in +pycodestyle==2.11.1 + # via autopep8 +pycparser==2.22 + # via cffi +pydantic==2.7.3 + # via + # antsibull-core + # antsibull-docs + # rstcheck-core +pydantic-core==2.18.4 + # via pydantic +pygments==2.18.0 + # via + # ansible-pygments + # rich + # sphinx +pylint==3.2.1 + # via -r requirements.in +pyproject-hooks==1.1.0 + # via + # build + # pip-tools pytest==8.2.0 + # via + # -r requirements.in + # pytest-fixture-config + # pytest-shutil + # pytest-virtualenv +pytest-fixture-config==1.7.0 + # via pytest-virtualenv +pytest-shutil==1.7.0 + # via pytest-virtualenv pytest-virtualenv==1.7.0 -docker==7.0.0 - -# Utility packages used in scripts + # via -r requirements.in pyyaml==6.0.1 -packaging==24.0 - -# Also include a version of ansible-core for IDE hints -# and as the default version used in tests. -# It is also needed for docs generation -ansible-core==2.16.6 - -# Docs -antsibull-docs==2.11.0 -ansible-pygments==0.1.1 + # via + # -r requirements.in + # ansible-compat + # ansible-core + # ansible-lint + # antsibull-changelog + # antsibull-core + # antsibull-docs + # molecule + # pre-commit + # yamllint +referencing==0.35.1 + # via + # jsonschema + # jsonschema-specifications +requests==2.32.3 + # via + # docker + # molecule-plugins + # sphinx +resolvelib==1.0.1 + # via ansible-core +rich==13.7.1 + # via + # ansible-lint + # enrich + # molecule + # typer +rpds-py==0.18.1 + # via + # jsonschema + # referencing +rstcheck==6.2.1 + # via + # antsibull-changelog + # antsibull-docs +rstcheck-core==1.2.1 + # via rstcheck +ruamel-yaml==0.18.6 + # via ansible-lint +ruamel-yaml-clib==0.2.8 + # via ruamel-yaml +selinux==0.3.0 + # via molecule-plugins +semantic-version==2.10.0 + # via + # antsibull-changelog + # antsibull-core + # antsibull-docs +shellingham==1.5.4 + # via typer +six==1.16.0 + # via + # pytest-shutil + # twiggy +snowballstemmer==2.2.0 + # via sphinx sphinx==7.3.7 + # via + # -r requirements.in + # antsibull-docs + # sphinx-ansible-theme + # sphinx-rtd-theme + # sphinxcontrib-jquery sphinx-ansible-theme==0.10.3 + # via -r requirements.in +sphinx-rtd-theme==2.0.0 + # via sphinx-ansible-theme +sphinxcontrib-applehelp==1.0.8 + # via sphinx +sphinxcontrib-devhelp==1.0.6 + # via sphinx +sphinxcontrib-htmlhelp==2.0.5 + # via sphinx +sphinxcontrib-jquery==4.1 + # via sphinx-rtd-theme +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-qthelp==1.0.7 + # via sphinx +sphinxcontrib-serializinghtml==1.1.10 + # via sphinx +subprocess-tee==0.4.1 + # via + # ansible-compat + # ansible-lint +termcolor==2.4.0 + # via pytest-shutil +tomlkit==0.12.5 + # via pylint +twiggy==0.5.1 + # via + # antsibull-core + # antsibull-docs +typer[all]==0.12.3 + # via rstcheck +typing-extensions==4.12.1 + # via + # pydantic + # pydantic-core + # typer +urllib3==2.2.1 + # via + # docker + # requests +virtualenv==20.26.2 + # via + # pre-commit + # pytest-virtualenv +wcmatch==8.5.2 + # via + # ansible-lint + # molecule +wheel==0.43.0 + # via pip-tools +yamllint==1.35.1 + # via ansible-lint +yarl==1.9.4 + # via aiohttp +zipp==3.19.2 + # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# pip +# setuptools diff --git a/scripts/setup.sh b/scripts/setup.sh index bc12a9df..a16ce52b 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -11,8 +11,8 @@ source .venv/bin/activate printf "Installing development requirements..." python3 -m pip install --upgrade pip --quiet python3 -m pip install --quiet -r requirements.txt --upgrade -# Also install the scenario requirements so we can run them directly -python3 -m pip install --quiet -r tests/roles/requirements.txt --upgrade +python3 -m piptools sync --pip-args "--quiet" # ensure contents stay synced with lockfile + printf "OK\n" printf "Installing pre-commit hook..." diff --git a/tests/roles/requirements.txt b/tests/roles/requirements.txt deleted file mode 100644 index 578fa5bf..00000000 --- a/tests/roles/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -# Dependencies for executing the role scenarios. -molecule==6.0.2 -molecule-plugins[docker]==23.4.1 diff --git a/tests/roles/requirements.txt b/tests/roles/requirements.txt new file mode 120000 index 00000000..fd1efae7 --- /dev/null +++ b/tests/roles/requirements.txt @@ -0,0 +1 @@ +../../requirements.txt \ No newline at end of file