From 6d2344b6ffbef2e1ac95933b081ed487b78af0c6 Mon Sep 17 00:00:00 2001 From: Mathieu Kniewallner Date: Sat, 10 Aug 2024 14:38:37 +0200 Subject: [PATCH] feat: add back PEP 420 support behind feature flag (#808) * feat(cli): add `--experimental-namespace-package` option * test(core): more robust local modules tests * feat(core): handle subdirectories in PEP 420 mode * test(functional): add tests for namespaced package * docs(usage): document `--experimental-namespace-package` --- docs/usage.md | 25 +++++++ python/deptry/cli.py | 8 +++ python/deptry/core.py | 19 ++++- tests/data/project_using_namespace/.gitignore | 1 + .../project_using_namespace/foo/api/http.py | 0 .../foo/database/bar.py | 7 ++ .../project_using_namespace/pyproject.toml | 17 +++++ tests/functional/cli/test_cli_namespace.py | 70 +++++++++++++++++++ tests/functional/utils.py | 1 + tests/unit/deprecate/test_requirements_txt.py | 1 + tests/unit/test_core.py | 64 +++++++++++++++-- 11 files changed, 205 insertions(+), 8 deletions(-) create mode 100644 tests/data/project_using_namespace/.gitignore create mode 100644 tests/data/project_using_namespace/foo/api/http.py create mode 100644 tests/data/project_using_namespace/foo/database/bar.py create mode 100644 tests/data/project_using_namespace/pyproject.toml create mode 100644 tests/functional/cli/test_cli_namespace.py diff --git a/docs/usage.md b/docs/usage.md index 1d1fe138..8b17e7af 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -489,3 +489,28 @@ pep621_dev_dependency_groups = ["test", "docs"] ```shell deptry . --pep621-dev-dependency-groups "test,docs" ``` + +#### Experimental namespace package + +!!! warning + This option is experimental and disabled by default for now, as it could degrade performance in large codebases. + +Enable experimental namespace package ([PEP 420](https://peps.python.org/pep-0420/)) support. + +When enabled, deptry will not only rely on the presence of `__init__.py` file in a directory to determine if it is a +local Python module or not, but will consider any Python file in the directory or its subdirectories, recursively. If a +Python file is found, then the directory will be considered as a local Python module. + +- Type: `bool` +- Default: `False` +- `pyproject.toml` option name: `experimental_namespace_package` +- CLI option name: `--experimental-namespace-package` +- `pyproject.toml` example: +```toml +[tool.deptry] +experimental_namespace_package = true +``` +- CLI example: +```shell +deptry . --experimental-namespace-package +``` diff --git a/python/deptry/cli.py b/python/deptry/cli.py index f8dfb33f..8b483c83 100644 --- a/python/deptry/cli.py +++ b/python/deptry/cli.py @@ -245,6 +245,11 @@ def display_deptry_version(ctx: click.Context, _param: click.Parameter, value: b default=(), show_default=False, ) +@click.option( + "--experimental-namespace-package", + is_flag=True, + help="Enable experimental support for namespace package (PEP 420) when detecting local modules (https://peps.python.org/pep-0420/).", +) def deptry( root: tuple[Path, ...], config: Path, @@ -262,6 +267,7 @@ def deptry( json_output: str, package_module_name_map: MutableMapping[str, tuple[str, ...]], pep621_dev_dependency_groups: tuple[str, ...], + experimental_namespace_package: bool, ) -> None: """Find dependency issues in your Python project. @@ -282,6 +288,7 @@ def deptry( if requirements_txt_dev: logging.warning(REQUIREMENTS_TXT_DEV_DEPRECATION_MESSAGE) + Core( root=root, config=config, @@ -299,4 +306,5 @@ def deptry( json_output=json_output, package_module_name_map=package_module_name_map, pep621_dev_dependency_groups=pep621_dev_dependency_groups, + experimental_namespace_package=experimental_namespace_package, ).run() diff --git a/python/deptry/core.py b/python/deptry/core.py index cd39de5b..43ee2a5c 100644 --- a/python/deptry/core.py +++ b/python/deptry/core.py @@ -1,6 +1,7 @@ from __future__ import annotations import logging +import os import sys from dataclasses import dataclass from typing import TYPE_CHECKING @@ -40,6 +41,7 @@ class Core: json_output: str package_module_name_map: Mapping[str, tuple[str, ...]] pep621_dev_dependency_groups: tuple[str, ...] + experimental_namespace_package: bool def run(self) -> None: self._log_config() @@ -116,14 +118,25 @@ def _get_local_modules(self) -> set[str]: return guessed_local_modules | set(self.known_first_party) - @staticmethod - def _is_local_module(path: Path) -> bool: + def _is_local_module(self, path: Path) -> bool: """Guess if a module is a local Python module.""" return bool( (path.is_file() and path.name != "__init__.py" and path.suffix == ".py") - or (path.is_dir() and list(path.glob("*.py"))) + or (path.is_dir() and self._directory_has_python_files(path)) ) + def _directory_has_python_files(self, path: Path) -> bool: + """Check if there is any Python file in the current directory. If experimental support for namespace packages + (PEP 420) is enabled, also search for Python files in subdirectories.""" + if self.experimental_namespace_package: + for _root, _dirs, files in os.walk(path): + for file in files: + if file.endswith(".py"): + return True + return False + + return bool(list(path.glob("*.py"))) + @staticmethod def _get_standard_library_modules() -> frozenset[str]: if sys.version_info[:2] >= (3, 10): diff --git a/tests/data/project_using_namespace/.gitignore b/tests/data/project_using_namespace/.gitignore new file mode 100644 index 00000000..796b96d1 --- /dev/null +++ b/tests/data/project_using_namespace/.gitignore @@ -0,0 +1 @@ +/build diff --git a/tests/data/project_using_namespace/foo/api/http.py b/tests/data/project_using_namespace/foo/api/http.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/project_using_namespace/foo/database/bar.py b/tests/data/project_using_namespace/foo/database/bar.py new file mode 100644 index 00000000..e18e7c8c --- /dev/null +++ b/tests/data/project_using_namespace/foo/database/bar.py @@ -0,0 +1,7 @@ +from os import chdir, walk +from pathlib import Path + +import flake8 +import white as w + +from foo import api diff --git a/tests/data/project_using_namespace/pyproject.toml b/tests/data/project_using_namespace/pyproject.toml new file mode 100644 index 00000000..ffad3ba3 --- /dev/null +++ b/tests/data/project_using_namespace/pyproject.toml @@ -0,0 +1,17 @@ +[project] +# PEP 621 project metadata +# See https://www.python.org/dev/peps/pep-0621/ +name = "foo" +version = "1.2.3" +requires-python = ">=3.7" +dependencies = ["tomli==2.0.1"] + +[project.optional-dependencies] +dev = ["flake8==7.1.1"] + +[build-system] +requires = ["setuptools>=61.0.0"] +build-backend = "setuptools.build_meta" + +[tool.deptry] +pep621_dev_dependency_groups = ["dev"] diff --git a/tests/functional/cli/test_cli_namespace.py b/tests/functional/cli/test_cli_namespace.py new file mode 100644 index 00000000..42f01eb6 --- /dev/null +++ b/tests/functional/cli/test_cli_namespace.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +import uuid +from pathlib import Path +from typing import TYPE_CHECKING + +import pytest + +from tests.functional.utils import Project +from tests.utils import get_issues_report + +if TYPE_CHECKING: + from tests.utils import PipVenvFactory + + +@pytest.mark.xdist_group(name=Project.NAMESPACE) +def test_cli_with_namespace(pip_venv_factory: PipVenvFactory) -> None: + with pip_venv_factory(Project.NAMESPACE) as virtual_env: + issue_report = f"{uuid.uuid4()}.json" + result = virtual_env.run(f"deptry . --experimental-namespace-package -o {issue_report}") + + assert result.returncode == 1 + assert get_issues_report(Path(issue_report)) == [ + { + "error": {"code": "DEP004", "message": "'flake8' imported but declared as a dev dependency"}, + "module": "flake8", + "location": {"file": str(Path("foo/database/bar.py")), "line": 4, "column": 8}, + }, + { + "error": {"code": "DEP001", "message": "'white' imported but missing from the dependency definitions"}, + "module": "white", + "location": {"file": str(Path("foo/database/bar.py")), "line": 5, "column": 8}, + }, + { + "error": {"code": "DEP002", "message": "'tomli' defined as a dependency but not used in the codebase"}, + "module": "tomli", + "location": {"file": str(Path("pyproject.toml")), "line": None, "column": None}, + }, + ] + + +@pytest.mark.xdist_group(name=Project.NAMESPACE) +def test_cli_with_namespace_without_experimental_flag(pip_venv_factory: PipVenvFactory) -> None: + with pip_venv_factory(Project.NAMESPACE) as virtual_env: + issue_report = f"{uuid.uuid4()}.json" + result = virtual_env.run(f"deptry . -o {issue_report}") + + assert result.returncode == 1 + assert get_issues_report(Path(issue_report)) == [ + { + "error": {"code": "DEP004", "message": "'flake8' imported but declared as a dev dependency"}, + "module": "flake8", + "location": {"file": str(Path("foo/database/bar.py")), "line": 4, "column": 8}, + }, + { + "error": {"code": "DEP001", "message": "'white' imported but missing from the dependency definitions"}, + "module": "white", + "location": {"file": str(Path("foo/database/bar.py")), "line": 5, "column": 8}, + }, + { + "error": {"code": "DEP003", "message": "'foo' imported but it is a transitive dependency"}, + "module": "foo", + "location": {"file": str(Path("foo/database/bar.py")), "line": 7, "column": 1}, + }, + { + "error": {"code": "DEP002", "message": "'tomli' defined as a dependency but not used in the codebase"}, + "module": "tomli", + "location": {"file": str(Path("pyproject.toml")), "line": None, "column": None}, + }, + ] diff --git a/tests/functional/utils.py b/tests/functional/utils.py index 08853829..cee3fd29 100644 --- a/tests/functional/utils.py +++ b/tests/functional/utils.py @@ -11,6 +11,7 @@ class Project(str, Enum): FUTURE_DEPRECATED_OBSOLETE_ARGUMENT = "project_with_future_deprecated_obsolete_argument" GITIGNORE = "project_with_gitignore" MULTIPLE_SOURCE_DIRECTORIES = "project_with_multiple_source_directories" + NAMESPACE = "project_using_namespace" PDM = "project_with_pdm" POETRY = "project_with_poetry" PYPROJECT_DIFFERENT_DIRECTORY = "project_with_pyproject_different_directory" diff --git a/tests/unit/deprecate/test_requirements_txt.py b/tests/unit/deprecate/test_requirements_txt.py index 8379aae2..51c61de6 100644 --- a/tests/unit/deprecate/test_requirements_txt.py +++ b/tests/unit/deprecate/test_requirements_txt.py @@ -25,6 +25,7 @@ "package_module_name_map": ANY, "pep621_dev_dependency_groups": ANY, "using_default_requirements_files": ANY, + "experimental_namespace_package": ANY, } diff --git a/tests/unit/test_core.py b/tests/unit/test_core.py index 57d7cd8a..ae67740b 100644 --- a/tests/unit/test_core.py +++ b/tests/unit/test_core.py @@ -25,30 +25,83 @@ @pytest.mark.parametrize( - ("known_first_party", "root_suffix", "expected"), + ("known_first_party", "root_suffix", "experimental_namespace_package", "expected"), [ ( (), "", - {"module_with_init", "module_without_init", "local_file"}, + False, + { + "local_file", + "module_with_init", + "module_without_init", + }, ), ( ("module_with_init", "module_without_init"), "", - {"module_with_init", "module_without_init", "local_file"}, + False, + { + "local_file", + "module_with_init", + "module_without_init", + }, ), ( ("module_without_init",), "module_with_init", - {"foo", "module_without_init", "subdirectory"}, + False, + { + "foo", + "module_without_init", + "subdirectory", + }, + ), + ( + (), + "", + True, + { + "local_file", + "module_using_namespace", + "module_with_init", + "module_without_init", + }, + ), + ( + ("module_with_init", "module_without_init"), + "", + True, + { + "local_file", + "module_using_namespace", + "module_with_init", + "module_without_init", + }, + ), + ( + ("module_without_init",), + "module_with_init", + True, + { + "foo", + "module_without_init", + "subdirectory", + }, ), ], ) def test__get_local_modules( - tmp_path: Path, known_first_party: tuple[str, ...], root_suffix: str, expected: set[str] + tmp_path: Path, + known_first_party: tuple[str, ...], + root_suffix: str, + experimental_namespace_package: bool, + expected: set[str], ) -> None: with run_within_dir(tmp_path): create_files([ + Path("directory_without_python_files/foo.txt"), + Path("module_using_namespace/subdirectory_namespace/foo.py"), Path("module_with_init/__init__.py"), Path("module_with_init/foo.py"), Path("module_with_init/subdirectory/__init__.py"), @@ -75,6 +128,7 @@ def test__get_local_modules( package_module_name_map={}, pep621_dev_dependency_groups=(), using_default_requirements_files=True, + experimental_namespace_package=experimental_namespace_package, )._get_local_modules() == expected )