diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index febb0ba1c..306754f86 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -24,7 +24,7 @@ jobs: - Windows - MacOs py: - - "3.10.0-rc.1" + - "3.10.0-rc.2" - "3.9" - "3.8" - "3.7" diff --git a/docs/changelog/2204.feature.rst b/docs/changelog/2204.feature.rst new file mode 100644 index 000000000..718779d0a --- /dev/null +++ b/docs/changelog/2204.feature.rst @@ -0,0 +1,2 @@ +Add ``external`` package type for :ref:`package` (see :ref:`external-package-builder`), and extract package dependencies +for packages passed in via :ref:`--installpkg ` - by :user:`gaborbernat`. diff --git a/docs/config.rst b/docs/config.rst index eb8cbca75..d9fbb94fc 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -476,6 +476,7 @@ Package Indicates where the packaging root file exists (historically setup.py file or pyproject.toml now). +.. _python-options: Python options ~~~~~~~~~~~~~~ @@ -542,8 +543,8 @@ Python run :keys: package :version_added: 4.0 - When option can be one of ``skip``, ``dev-legacy``, ``sdist`` or ``wheel``. If :ref:`use_develop` is set this becomes - a constant of ``dev-legacy``. If :ref:`skip_install` is set this becomes a constant of ``skip``. + When option can be one of ``skip``, ``dev-legacy``, ``sdist``, ``wheel`` or ``external``. If :ref:`use_develop` is + set this becomes a constant of ``dev-legacy``. If :ref:`skip_install` is set this becomes a constant of ``skip``. .. conf:: @@ -565,6 +566,59 @@ Python run A list of "extras" from the package to be installed. For example, ``extras = testing`` is equivalent to ``[testing]`` in a ``pip install`` command. +.. _external-package-builder: + +External package builder +~~~~~~~~~~~~~~~~~~~~~~~~ + +tox supports operating with externally built packages. External packages might be provided in two wayas: + +- explicitly via the :ref:`--installpkg ` CLI argument, +- setting the :ref:`package` to ``external`` and using a tox packaging environment named ``_external`` + (see :ref:`package_env`) to build the package. The tox packaging environment takes all configuration flags of a + :ref:`python environment `, plus the following: + +.. conf:: + :keys: deps + :default: + :ref_suffix: external + + Name of the Python dependencies as specified by `PEP-440`_. Installed into the environment prior running the build + commands. All installer commands are executed using the :ref:`tox_root` as the current working directory. + +.. conf:: + :keys: commands + :default: + :ref_suffix: external + + Commands to run that will build the package. If any command fails the packaging operation is considered failed and + will fail all environments using that package. + +.. conf:: + :keys: ignore_errors + :default: False + :ref_suffix: external + + When executing the commands keep going even if a sub-command exits with non-zero exit code. The overall status will + be "commands failed", i.e. tox will exit non-zero in case any command failed. It may be helpful to note that this + setting is analogous to the ``-k`` or ``--keep-going`` option of GNU Make. + +.. conf:: + :keys: change_dir, changedir + :default: {tox root} + :ref_suffix: external + + Change to this working directory when executing the package build command. If the directory does not exist yet, it + will be created (required for Windows to be able to execute any command). + +.. conf:: + :keys: package_glob + :default: {envtmpdir}{/}dist{/}* + + A glob that should match the wheel/sdist file to install. If no file or multiple files is matched the packaging + operation is considered failed and will raise an error. + + Python virtual environment ~~~~~~~~~~~~~~~~~~~~~~~~~~ .. conf:: diff --git a/src/tox/execute/pep517_backend.py b/src/tox/execute/pep517_backend.py index 4bd26b875..048ae7e8c 100644 --- a/src/tox/execute/pep517_backend.py +++ b/src/tox/execute/pep517_backend.py @@ -52,7 +52,7 @@ def local_execute(self, options: ExecuteOptions) -> Tuple[LocalSubProcessExecute self.is_alive = True break if b"failed to start backend" in status.err: - from tox.tox_env.python.virtual_env.package.api import ToxBackendFailed + from tox.tox_env.python.virtual_env.package.pep517 import ToxBackendFailed failure = BackendFailed( result={ diff --git a/src/tox/plugin/manager.py b/src/tox/plugin/manager.py index ad74b5611..02dbac24e 100644 --- a/src/tox/plugin/manager.py +++ b/src/tox/plugin/manager.py @@ -14,7 +14,7 @@ from tox.session.cmd.run import parallel, sequential from tox.tox_env import package as package_api from tox.tox_env.python.virtual_env import runner -from tox.tox_env.python.virtual_env.package import api +from tox.tox_env.python.virtual_env.package import cmd_builder, pep517 from tox.tox_env.register import REGISTER, ToxEnvRegister from ..config.main import Config @@ -33,7 +33,8 @@ def __init__(self) -> None: loader_api, provision, runner, - api, + pep517, + cmd_builder, legacy, version_flag, exec_, diff --git a/src/tox/pytest.py b/src/tox/pytest.py index a075d02dd..3f054ff8f 100644 --- a/src/tox/pytest.py +++ b/src/tox/pytest.py @@ -301,19 +301,19 @@ def enable_pep517_backend_coverage() -> Iterator[None]: # noqa: PT004 yield # pragma: no cover return # pragma: no cover # the COV_ env variables needs to be passed on for the PEP-517 backend - from tox.tox_env.python.virtual_env.package.api import Pep517VirtualEnvPackage + from tox.tox_env.python.virtual_env.package.pep517 import Pep517VirtualEnvPackager - def default_pass_env(self: Pep517VirtualEnvPackage) -> List[str]: + def default_pass_env(self: Pep517VirtualEnvPackager) -> List[str]: result = previous(self) result.append("COV_*") return result - previous = Pep517VirtualEnvPackage._default_pass_env + previous = Pep517VirtualEnvPackager._default_pass_env try: - Pep517VirtualEnvPackage._default_pass_env = default_pass_env # type: ignore + Pep517VirtualEnvPackager._default_pass_env = default_pass_env # type: ignore yield finally: - Pep517VirtualEnvPackage._default_pass_env = previous # type: ignore + Pep517VirtualEnvPackager._default_pass_env = previous # type: ignore class ToxRunOutcome: diff --git a/src/tox/session/cmd/run/single.py b/src/tox/session/cmd/run/single.py index 4d6db52e4..f8dd9412a 100644 --- a/src/tox/session/cmd/run/single.py +++ b/src/tox/session/cmd/run/single.py @@ -8,8 +8,9 @@ from tox.config.types import Command from tox.execute.api import Outcome, StdinSource +from tox.tox_env.api import ToxEnv from tox.tox_env.errors import Fail, Skip -from tox.tox_env.python.virtual_env.package.api import ToxBackendFailed +from tox.tox_env.python.virtual_env.package.pep517 import ToxBackendFailed from tox.tox_env.runner import RunToxEnv LOGGER = logging.getLogger(__name__) @@ -86,7 +87,7 @@ def run_commands(tox_env: RunToxEnv, no_test: bool) -> Tuple[int, List[Outcome]] return exit_code, outcomes -def run_command_set(tox_env: RunToxEnv, key: str, cwd: Path, ignore_errors: bool, outcomes: List[Outcome]) -> int: +def run_command_set(tox_env: ToxEnv, key: str, cwd: Path, ignore_errors: bool, outcomes: List[Outcome]) -> int: exit_code = Outcome.OK command_set: List[Command] = tox_env.conf[key] for at, cmd in enumerate(command_set): @@ -113,5 +114,6 @@ def run_command_set(tox_env: RunToxEnv, key: str, cwd: Path, ignore_errors: bool __all__ = ( "run_one", + "run_command_set", "ToxEnvRunResult", ) diff --git a/src/tox/session/state.py b/src/tox/session/state.py index 1501ba7e0..d47daad97 100644 --- a/src/tox/session/state.py +++ b/src/tox/session/state.py @@ -79,10 +79,24 @@ def _build_run_env(self, env_conf: EnvConfigSet) -> None: self._build_package_env(env) def _build_package_env(self, env: RunToxEnv) -> None: - for tag, name, core_type in env.iter_package_env_types(): - with self.log_handler.with_context(name): - package_tox_env = self._get_package_env(core_type, name) - env.notify_of_package_env(tag, package_tox_env) + pkg_info = env.get_package_env_types() + if pkg_info is not None: + name, core_type = pkg_info + env.package_env = self._build_pkg_env(name, core_type, env) + + def _build_pkg_env(self, name: str, core_type: str, env: RunToxEnv) -> PackageToxEnv: + with self.log_handler.with_context(name): + package_tox_env = self._get_package_env(core_type, name) + + child_package_envs = package_tox_env.register_run_env(env) + try: + child_name, child_type = next(child_package_envs) + while True: + child_pkg_env = self._build_pkg_env(child_name, child_type, env) + child_name, child_type = child_package_envs.send(child_pkg_env) + except StopIteration: + pass + return package_tox_env def _get_package_env(self, packager: str, name: str) -> PackageToxEnv: if name in self._pkg_env: # if already created reuse diff --git a/src/tox/tox_env/api.py b/src/tox/tox_env/api.py index aed536e5d..fd28c7c0c 100644 --- a/src/tox/tox_env/api.py +++ b/src/tox/tox_env/api.py @@ -284,7 +284,7 @@ def _clean(self, transitive: bool = False) -> None: # noqa: U100 self._run_state.update({"setup": False, "clean": True}) @property - def _environment_variables(self) -> Dict[str, str]: + def environment_variables(self) -> Dict[str, str]: pass_env: List[str] = self.conf["pass_env"] set_env: SetEnv = self.conf["set_env"] if self._env_vars_pass_env == pass_env and not set_env.changed and self._env_vars is not None: @@ -372,7 +372,7 @@ def execute_async( cwd = self.core["tox_root"] if show is None: show = self.options.verbosity > 3 - request = ExecuteRequest(cmd, cwd, self._environment_variables, stdin, run_id, allow=self._allow_externals) + request = ExecuteRequest(cmd, cwd, self.environment_variables, stdin, run_id, allow=self._allow_externals) if _CWD == request.cwd: repr_cwd = "" else: diff --git a/src/tox/tox_env/package.py b/src/tox/tox_env/package.py index 647c162d8..145eff725 100644 --- a/src/tox/tox_env/package.py +++ b/src/tox/tox_env/package.py @@ -4,13 +4,16 @@ from abc import ABC, abstractmethod from pathlib import Path from threading import Lock -from typing import List, Optional, Set, cast +from typing import TYPE_CHECKING, Generator, Iterator, List, Optional, Set, Tuple, cast from tox.config.main import Config from tox.config.sets import EnvConfigSet from .api import ToxEnv, ToxEnvCreateArgs +if TYPE_CHECKING: + from .runner import RunToxEnv + class Package: """package""" @@ -53,13 +56,19 @@ def _recreate_default(self, conf: "Config", value: Optional[str]) -> bool: def perform_packaging(self, for_env: EnvConfigSet) -> List[Package]: raise NotImplementedError - def notify_of_run_env(self, conf: EnvConfigSet) -> None: - with self._lock: - self._envs.add(conf.name) - def teardown_env(self, conf: EnvConfigSet) -> None: with self._lock: self._envs.remove(conf.name) has_envs = bool(self._envs) if not has_envs: self._teardown() + + def register_run_env(self, run_env: "RunToxEnv") -> Generator[Tuple[str, str], "PackageToxEnv", None]: + with self._lock: + self._envs.add(run_env.conf.name) + return + yield # make this a generator + + @abstractmethod + def child_pkg_envs(self, run_conf: EnvConfigSet) -> Iterator["PackageToxEnv"]: + raise NotImplementedError diff --git a/src/tox/tox_env/python/package.py b/src/tox/tox_env/python/package.py index 5964bcafc..2d43a96f1 100644 --- a/src/tox/tox_env/python/package.py +++ b/src/tox/tox_env/python/package.py @@ -3,12 +3,19 @@ """ from abc import ABC, abstractmethod from pathlib import Path -from typing import Any, Sequence, Tuple +from typing import TYPE_CHECKING, Any, Dict, Generator, Iterator, Optional, Sequence, Tuple, Union, cast from packaging.requirements import Requirement +from ...config.sets import EnvConfigSet +from ..api import ToxEnvCreateArgs from ..package import Package, PackageToxEnv, PathPackage +from ..runner import RunToxEnv from .api import Python +from .pip.req_file import PythonDeps + +if TYPE_CHECKING: + from tox.config.main import Config class PythonPackage(Package): @@ -34,6 +41,10 @@ class DevLegacyPackage(PythonPathPackageWithDeps): class PythonPackageToxEnv(Python, PackageToxEnv, ABC): + def __init__(self, create_args: ToxEnvCreateArgs) -> None: + self._wheel_build_envs: Dict[str, PythonPackageToxEnv] = {} + super().__init__(create_args) + def register_config(self) -> None: super().register_config() @@ -43,5 +54,47 @@ def _setup_env(self) -> None: self.installer.install(self.requires(), PythonPackageToxEnv.__name__, "requires") @abstractmethod - def requires(self) -> Tuple[Requirement, ...]: + def requires(self) -> Union[Tuple[Requirement, ...], PythonDeps]: raise NotImplementedError + + def register_run_env(self, run_env: RunToxEnv) -> Generator[Tuple[str, str], PackageToxEnv, None]: + yield from super().register_run_env(run_env) + if not isinstance(run_env, Python) or run_env.conf["package"] != "wheel" or "wheel_build_env" in run_env.conf: + return + + def default_wheel_tag(conf: "Config", env_name: Optional[str]) -> str: + # https://www.python.org/dev/peps/pep-0427/#file-name-convention + # when building wheels we need to ensure that the built package is compatible with the target env + # compatibility is documented within https://www.python.org/dev/peps/pep-0427/#file-name-convention + # a wheel tag example: {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl + # python only code are often compatible at major level (unless universal wheel in which case both 2/3) + # c-extension codes are trickier, but as of today both poetry/setuptools uses pypa/wheels logic + # https://github.com/pypa/wheel/blob/master/src/wheel/bdist_wheel.py#L234-L280 + run_py = cast(Python, run_env).base_python + if run_py is None: + raise ValueError(f"could not resolve base python for {self.conf.name}") + + default_pkg_py = self.base_python + if ( + default_pkg_py.version_no_dot == run_py.version_no_dot + and default_pkg_py.impl_lower == run_py.impl_lower + ): + return self.conf.name + + return f"{self.conf.name}-{run_py.impl_lower}{run_py.version_no_dot}" + + run_env.conf.add_config( + keys=["wheel_build_env"], + of_type=str, + default=default_wheel_tag, + desc="wheel tag to use for building applications", + ) + pkg_env = run_env.conf["wheel_build_env"] + result = yield pkg_env, run_env.conf["package_tox_env_type"] + self._wheel_build_envs[pkg_env] = cast(PythonPackageToxEnv, result) + + def child_pkg_envs(self, run_conf: EnvConfigSet) -> Iterator[PackageToxEnv]: + if run_conf["package"] == "wheel": + env = self._wheel_build_envs.get(run_conf["wheel_build_env"]) + if env is not None and env.name != self.name: + yield env diff --git a/src/tox/tox_env/python/pip/pip_install.py b/src/tox/tox_env/python/pip/pip_install.py index f5f2b3104..456b6cc84 100644 --- a/src/tox/tox_env/python/pip/pip_install.py +++ b/src/tox/tox_env/python/pip/pip_install.py @@ -135,8 +135,6 @@ def _install_list_of_deps( elif isinstance(arg, DevLegacyPackage): groups["req"].extend(str(i) for i in arg.deps) groups["dev_pkg"].append(str(arg.path)) - elif isinstance(arg, PathPackage): - groups["path_pkg"].append(str(arg.path)) else: logging.warning(f"pip cannot install {arg!r}") raise SystemExit(1) @@ -158,8 +156,6 @@ def _install_list_of_deps( for entry in groups["dev_pkg"]: install_args.extend(("-e", str(entry))) self._execute_installer(install_args, of_type) - if groups["path_pkg"]: - self._execute_installer(groups["path_pkg"], of_type) def _execute_installer(self, deps: Sequence[Any], of_type: str) -> None: cmd = self.build_install_cmd(deps) diff --git a/src/tox/tox_env/python/pip/req/file.py b/src/tox/tox_env/python/pip/req/file.py index f5e5e5432..1e208837b 100644 --- a/src/tox/tox_env/python/pip/req/file.py +++ b/src/tox/tox_env/python/pip/req/file.py @@ -56,7 +56,9 @@ def __init__(self, req: str, options: Dict[str, Any], from_file: str, lineno: in else: path = root / req extra_part = f"[{','.join(sorted(extras))}]" if extras else "" - rel_path = path.resolve().relative_to(root) + rel_path = str(path.resolve().relative_to(root)) + if rel_path != "." and os.sep not in rel_path: # prefix paths in cwd to not convert them to requirement + rel_path = f".{os.sep}{rel_path}" self._requirement = f"{rel_path}{extra_part}" self._options = options self._from_file = from_file diff --git a/src/tox/tox_env/python/pip/req_file.py b/src/tox/tox_env/python/pip/req_file.py index ab0b41940..7d1dd1f20 100644 --- a/src/tox/tox_env/python/pip/req_file.py +++ b/src/tox/tox_env/python/pip/req_file.py @@ -77,6 +77,12 @@ def unroll(self) -> Tuple[List[str], List[str]]: self._unroll = result_opts, result_req return self._unroll + @classmethod + def factory(cls, root: Path, raw: object) -> "PythonDeps": + if not isinstance(raw, str): + raise TypeError(raw) + return cls(raw, root) + ONE_ARG = { "-i", diff --git a/src/tox/tox_env/python/runner.py b/src/tox/tox_env/python/runner.py index b0da26728..e68366c43 100644 --- a/src/tox/tox_env/python/runner.py +++ b/src/tox/tox_env/python/runner.py @@ -2,14 +2,12 @@ A tox run environment that handles the Python language. """ from abc import ABC -from pathlib import Path -from typing import Iterator, List, Optional, Set, Tuple +from functools import partial +from typing import List, Set, Tuple -from tox.config.main import Config from tox.report import HandledError from tox.tox_env.errors import Skip -from tox.tox_env.package import Package, PathPackage -from tox.tox_env.python.package import PythonPackageToxEnv +from tox.tox_env.package import Package from tox.tox_env.python.pip.req_file import PythonDeps from ..api import ToxEnvCreateArgs @@ -24,16 +22,10 @@ def __init__(self, create_args: ToxEnvCreateArgs) -> None: def register_config(self) -> None: super().register_config() root = self.core["toxinidir"] - - def python_deps_factory(raw: object) -> PythonDeps: - if not isinstance(raw, str): - raise TypeError(raw) - return PythonDeps(raw, root) - self.conf.add_config( keys="deps", of_type=PythonDeps, - factory=python_deps_factory, + factory=partial(PythonDeps.factory, root), default=PythonDeps("", root), desc="Name of the python dependencies as specified by PEP-440", ) @@ -44,63 +36,31 @@ def python_deps_factory(raw: object) -> PythonDeps: desc="skip running missing interpreters", ) - def iter_package_env_types(self) -> Iterator[Tuple[str, str, str]]: - yield from super().iter_package_env_types() - if self.pkg_type == "wheel": - wheel_build_env: str = self.conf["wheel_build_env"] - if wheel_build_env not in self._package_envs: # pragma: no branch - package_tox_env_type = self.conf["package_tox_env_type"] - yield "wheel", wheel_build_env, package_tox_env_type - @property def _package_types(self) -> Tuple[str, ...]: - return "wheel", "sdist", "dev-legacy", "skip" + return "wheel", "sdist", "dev-legacy", "skip", "external" def _register_package_conf(self) -> bool: + # provision package type desc = f"package installation mode - {' | '.join(i for i in self._package_types)} " if not super()._register_package_conf(): self.conf.add_constant(["package"], desc, "skip") return False - self.conf.add_config(keys=["use_develop", "usedevelop"], desc="use develop mode", default=False, of_type=bool) - develop_mode = self.conf["use_develop"] or getattr(self.options, "develop", False) - if develop_mode: - self.conf.add_constant(["package"], desc, "dev-legacy") + if getattr(self.options, "install_pkg", None) is not None: + self.conf.add_constant(["package"], desc, "external") else: - self.conf.add_config(keys="package", of_type=str, default=self.default_pkg_type, desc=desc) - pkg_type = self.pkg_type + self.conf.add_config( + keys=["use_develop", "usedevelop"], desc="use develop mode", default=False, of_type=bool + ) + develop_mode = self.conf["use_develop"] or getattr(self.options, "develop", False) + if develop_mode: + self.conf.add_constant(["package"], desc, "dev-legacy") + else: + self.conf.add_config(keys="package", of_type=str, default=self.default_pkg_type, desc=desc) + pkg_type = self.pkg_type if pkg_type == "skip": return False - - if pkg_type == "wheel": - - def default_wheel_tag(conf: "Config", env_name: Optional[str]) -> str: - # https://www.python.org/dev/peps/pep-0427/#file-name-convention - # when building wheels we need to ensure that the built package is compatible with the target env - # compatibility is documented within https://www.python.org/dev/peps/pep-0427/#file-name-convention - # a wheel tag example: {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl - # python only code are often compatible at major level (unless universal wheel in which case both 2/3) - # c-extension codes are trickier, but as of today both poetry/setuptools uses pypa/wheels logic - # https://github.com/pypa/wheel/blob/master/src/wheel/bdist_wheel.py#L234-L280 - default_package_env = self._package_envs["default"] - self_py = self.base_python - if self_py is not None and isinstance(default_package_env, PythonPackageToxEnv): - default_pkg_py = default_package_env.base_python - if ( - default_pkg_py.version_no_dot == self_py.version_no_dot - and default_pkg_py.impl_lower == self_py.impl_lower - ): - return default_package_env.conf.name - if self_py is None: - raise ValueError(f"could not resolve base python for {self.conf.name}") - return f"{default_package_env.conf.name}-{self_py.impl_lower}{self_py.version_no_dot}" - - self.conf.add_config( - keys=["wheel_build_env"], - of_type=str, - default=default_wheel_tag, - desc="wheel tag to use for building applications", - ) self.conf.add_config( keys=["extras"], of_type=Set[str], @@ -130,17 +90,11 @@ def _install_deps(self) -> None: self.installer.install(requirements_file, PythonRun.__name__, "deps") def _build_packages(self) -> List[Package]: - explicit_install_package: Optional[Path] = getattr(self.options, "install_pkg", None) - if explicit_install_package is not None: - return [PathPackage(explicit_install_package)] - - package_env = self._package_envs[self._get_package_env()] + package_env = self.package_env + assert package_env is not None with package_env.display_context(self._has_display_suspended): try: packages = package_env.perform_packaging(self.conf) except Skip as exception: raise Skip(f"{exception.args[0]} for package environment {package_env.conf['env_name']}") return packages - - def _get_package_env(self) -> str: - return "wheel" if self.pkg_type == "wheel" else "default" diff --git a/src/tox/tox_env/python/virtual_env/api.py b/src/tox/tox_env/python/virtual_env/api.py index 94bd6d636..90bf3a6c3 100644 --- a/src/tox/tox_env/python/virtual_env/api.py +++ b/src/tox/tox_env/python/virtual_env/api.py @@ -35,7 +35,7 @@ def register_config(self) -> None: keys=["system_site_packages", "sitepackages"], of_type=bool, default=lambda conf, name: StrConvert().to_bool( - self._environment_variables.get("VIRTUALENV_SYSTEM_SITE_PACKAGES", "False") + self.environment_variables.get("VIRTUALENV_SYSTEM_SITE_PACKAGES", "False") ), desc="create virtual environments that also have access to globally installed packages.", ) @@ -43,8 +43,8 @@ def register_config(self) -> None: keys=["always_copy", "alwayscopy"], of_type=bool, default=lambda conf, name: StrConvert().to_bool( - self._environment_variables.get( - "VIRTUALENV_COPIES", self._environment_variables.get("VIRTUALENV_ALWAYS_COPY", "False") + self.environment_variables.get( + "VIRTUALENV_COPIES", self.environment_variables.get("VIRTUALENV_ALWAYS_COPY", "False") ) ), desc="force virtualenv to always copy rather than symlink", @@ -53,7 +53,7 @@ def register_config(self) -> None: keys=["download"], of_type=bool, default=lambda conf, name: StrConvert().to_bool( - self._environment_variables.get("VIRTUALENV_DOWNLOAD", "False") + self.environment_variables.get("VIRTUALENV_DOWNLOAD", "False") ), desc="true if you want virtualenv to upgrade pip/wheel/setuptools to the latest version", ) @@ -105,7 +105,7 @@ def session(self) -> Session: return self._virtualenv_session def virtualenv_env_vars(self) -> Dict[str, str]: - env = self._environment_variables.copy() + env = self.environment_variables.copy() base_python: List[str] = self.conf["base_python"] if "VIRTUALENV_NO_PERIODIC_UPDATE" not in env: env["VIRTUALENV_NO_PERIODIC_UPDATE"] = "True" diff --git a/src/tox/tox_env/python/virtual_env/package/cmd_builder.py b/src/tox/tox_env/python/virtual_env/package/cmd_builder.py new file mode 100644 index 000000000..b8fe923f4 --- /dev/null +++ b/src/tox/tox_env/python/virtual_env/package/cmd_builder.py @@ -0,0 +1,164 @@ +import glob +import shutil +import sys +import tarfile +from functools import partial +from io import TextIOWrapper +from os import PathLike +from pathlib import Path +from typing import Generator, Iterator, List, Optional, Set, Tuple, cast +from zipfile import ZipFile + +from packaging.requirements import Requirement + +from tox.config.sets import EnvConfigSet +from tox.config.types import Command +from tox.execute import Outcome +from tox.plugin import impl +from tox.session.cmd.run.single import run_command_set +from tox.tox_env.api import ToxEnvCreateArgs +from tox.tox_env.errors import Fail +from tox.tox_env.package import Package, PackageToxEnv +from tox.tox_env.python.package import PythonPackageToxEnv, SdistPackage, WheelPackage +from tox.tox_env.python.pip.req_file import PythonDeps +from tox.tox_env.python.virtual_env.api import VirtualEnv +from tox.tox_env.register import ToxEnvRegister +from tox.tox_env.runner import RunToxEnv + +from .pep517 import Pep517VirtualEnvPackager +from .util import dependencies_with_extras + +if sys.version_info >= (3, 8): # pragma: no cover (py38+) + from importlib.metadata import Distribution +else: # pragma: no cover (py38+) + from importlib_metadata import Distribution + + +class VirtualEnvCmdBuilder(PythonPackageToxEnv, VirtualEnv): + def __init__(self, create_args: ToxEnvCreateArgs) -> None: + super().__init__(create_args) + self._sdist_meta_tox_env: Optional[Pep517VirtualEnvPackager] = None + + @staticmethod + def id() -> str: + return "virtualenv-cmd-builder" + + def register_config(self) -> None: + super().register_config() + root = self.core["toxinidir"] + self.conf.add_config( + keys="deps", + of_type=PythonDeps, + factory=partial(PythonDeps.factory, root), + default=PythonDeps("", root), + desc="Name of the python dependencies as specified by PEP-440", + ) + self.conf.add_config( + keys=["commands"], + of_type=List[Command], + default=[], + desc="the commands to be called for testing", + ) + self.conf.add_config( + keys=["change_dir", "changedir"], + of_type=Path, + default=lambda conf, name: cast(Path, conf.core["tox_root"]), + desc="change to this working directory when executing the test command", + ) + self.conf.add_config( + keys=["ignore_errors"], + of_type=bool, + default=False, + desc="when executing the commands keep going even if a sub-command exits with non-zero exit code", + ) + self.conf.add_config( + keys=["package_glob"], + of_type=str, + default=str(self.conf["env_tmp_dir"] / "dist" / "*"), + desc="when executing the commands keep going even if a sub-command exits with non-zero exit code", + ) + + def requires(self) -> PythonDeps: + return cast(PythonDeps, self.conf["deps"]) + + def perform_packaging(self, for_env: EnvConfigSet) -> List[Package]: + self.setup() + path: Optional[Path] = getattr(self.options, "install_pkg", None) + if path is None: # use install_pkg if specified, otherwise build via commands + chdir: Path = self.conf["change_dir"] + ignore_errors: bool = self.conf["ignore_errors"] + status = run_command_set(self, "commands", chdir, ignore_errors, []) + if status != Outcome.OK: + raise Fail("stopping as failed to build package") + package_glob = self.conf["package_glob"] + found = glob.glob(package_glob) + if not found: + raise Fail(f"no package found in {package_glob}") + elif len(found) != 1: + raise Fail(f"found more than one package {', '.join(sorted(found))}") + path = Path(found[0]) + return self.extract_install_info(for_env, path) + + def extract_install_info(self, for_env: EnvConfigSet, path: Path) -> List[Package]: + extras: Set[str] = for_env["extras"] + if path.suffix == ".whl": + requires: List[str] = WheelDistribution(path).requires or [] + package: Package = WheelPackage(path, dependencies_with_extras([Requirement(i) for i in requires], extras)) + else: # must be source distribution + work_dir = self.env_tmp_dir / "sdist-extract" + if work_dir.exists(): # pragma: no branch + shutil.rmtree(work_dir) # pragma: no cover + work_dir.mkdir() + with tarfile.open(str(path), "r:gz") as tar: + tar.extractall(path=str(work_dir)) + assert self._sdist_meta_tox_env is not None # the register run env is guaranteed to be called before this + with self._sdist_meta_tox_env.display_context(self._has_display_suspended): + self._sdist_meta_tox_env.root = next(work_dir.iterdir()) # contains a single egg info folder + deps = self._sdist_meta_tox_env.get_package_dependencies() + package = SdistPackage(path, dependencies_with_extras(deps, extras)) + return [package] + + def register_run_env(self, run_env: RunToxEnv) -> Generator[Tuple[str, str], PackageToxEnv, None]: + yield from super().register_run_env(run_env) + # in case the outcome is a sdist we'll use this to find out its metadata + result = yield f"{self.conf.name}_sdist_meta", Pep517VirtualEnvPackager.id() + self._sdist_meta_tox_env = cast(Pep517VirtualEnvPackager, result) + + def child_pkg_envs(self, run_conf: EnvConfigSet) -> Iterator[PackageToxEnv]: # noqa: U100 + if self._sdist_meta_tox_env is not None: # pragma: no branch + yield self._sdist_meta_tox_env + + +class WheelDistribution(Distribution): # type: ignore # cannot subclass has type Any + def __init__(self, wheel: Path) -> None: + self._wheel = wheel + self._dist_name: Optional[str] = None + + @property + def dist_name(self) -> str: + if self._dist_name is None: + with ZipFile(self._wheel) as zip_file: + for name in zip_file.namelist(): + root = name.split("/")[0] + if root.endswith(".dist-info"): + self._dist_name = root + break + else: + raise Fail(f"no .dist-info inside {self._wheel}") + return self._dist_name + + def read_text(self, filename: str) -> Optional[str]: + with ZipFile(self._wheel) as zip_file: + try: + with TextIOWrapper(zip_file.open(f"{self.dist_name}/{filename}"), encoding="utf-8") as file_handler: + return file_handler.read() + except KeyError: + return None + + def locate_file(self, path: str) -> "PathLike[str]": + return self._wheel / path # pragma: no cover # not used by us, but part of the ABC + + +@impl +def tox_register_tox_env(register: ToxEnvRegister) -> None: + register.add_package_env(VirtualEnvCmdBuilder) diff --git a/src/tox/tox_env/python/virtual_env/package/api.py b/src/tox/tox_env/python/virtual_env/package/pep517.py similarity index 66% rename from src/tox/tox_env/python/virtual_env/package/api.py rename to src/tox/tox_env/python/virtual_env/package/pep517.py index 3ec36047c..758143c7f 100644 --- a/src/tox/tox_env/python/virtual_env/package/api.py +++ b/src/tox/tox_env/python/virtual_env/package/pep517.py @@ -1,13 +1,11 @@ import os import sys from contextlib import contextmanager -from copy import deepcopy from pathlib import Path from threading import RLock -from typing import Any, Dict, Iterator, List, NoReturn, Optional, Sequence, Set, Tuple, Union, cast +from typing import Any, Dict, Generator, Iterator, List, NoReturn, Optional, Sequence, Set, Tuple, cast from cachetools import cached -from packaging.markers import Variable from packaging.requirements import Requirement from tox.config.sets import EnvConfigSet @@ -17,12 +15,14 @@ from tox.plugin import impl from tox.tox_env.api import ToxEnvCreateArgs from tox.tox_env.errors import Fail -from tox.tox_env.package import Package +from tox.tox_env.package import Package, PackageToxEnv from tox.tox_env.python.package import DevLegacyPackage, PythonPackageToxEnv, SdistPackage, WheelPackage from tox.tox_env.register import ToxEnvRegister +from tox.tox_env.runner import RunToxEnv from tox.util.pep517.frontend import BackendFailed, CmdStatus, ConfigSettings, Frontend from ..api import VirtualEnv +from .util import dependencies_with_extras if sys.version_info >= (3, 8): # pragma: no cover (py38+) from importlib.metadata import Distribution, PathDistribution # type: ignore[attr-defined] @@ -30,9 +30,6 @@ from importlib_metadata import Distribution, PathDistribution -TOX_PACKAGE_ENV_ID = "virtualenv-pep-517" - - class ToxBackendFailed(Fail, BackendFailed): def __init__(self, backend_failed: BackendFailed) -> None: Fail.__init__(self) @@ -69,28 +66,28 @@ def out_err(self) -> Tuple[str, str]: return status.outcome.out_err() -class Pep517VirtualEnvPackage(PythonPackageToxEnv, VirtualEnv, Frontend): +class Pep517VirtualEnvPackager(PythonPackageToxEnv, VirtualEnv): """local file system python virtual environment via the virtualenv package""" def __init__(self, create_args: ToxEnvCreateArgs) -> None: - VirtualEnv.__init__(self, create_args) - root: Path = self.conf["package_root"] - Frontend.__init__(self, *Frontend.create_args_from_folder(root)) - - self._backend_executor_: Optional[LocalSubProcessPep517Executor] = None - self._builds: Set[str] = set() + super().__init__(create_args) + self.root: Path = self.conf["package_root"] + self._frontend_private: Optional[Pep517VirtualEnvFrontend] = None + self.builds: Set[str] = set() self._distribution_meta: Optional[PathDistribution] = None self._package_dependencies: Optional[List[Requirement]] = None self._pkg_lock = RLock() # can build only one package at a time - into: Dict[str, Any] = {} - pkg_cache = cached(into, key=lambda *args, **kwargs: "wheel" if "wheel_directory" in kwargs else "sdist") - self.build_wheel = pkg_cache(self.build_wheel) # type: ignore - self.build_sdist = pkg_cache(self.build_sdist) # type: ignore @staticmethod def id() -> str: return "virtualenv-pep-517" + @property + def _frontend(self) -> "Pep517VirtualEnvFrontend": + if self._frontend_private is None: + self._frontend_private = Pep517VirtualEnvFrontend(self.root, self) + return self._frontend_private + def register_config(self) -> None: super().register_config() self.conf.add_config( @@ -116,54 +113,70 @@ def meta_folder(self) -> Path: meta_folder.mkdir(exist_ok=True) return meta_folder - def notify_of_run_env(self, conf: EnvConfigSet) -> None: - super().notify_of_run_env(conf) - self._builds.add(conf["package"]) + def register_run_env(self, run_env: RunToxEnv) -> Generator[Tuple[str, str], PackageToxEnv, None]: + yield from super().register_run_env(run_env) + self.builds.add(run_env.conf["package"]) def _setup_env(self) -> None: super()._setup_env() - if "wheel" in self._builds: - build_requires = self.get_requires_for_build_wheel().requires + if "wheel" in self.builds: + build_requires = self._frontend.get_requires_for_build_wheel().requires self.installer.install(build_requires, PythonPackageToxEnv.__name__, "requires_for_build_wheel") - if "sdist" in self._builds: - build_requires = self.get_requires_for_build_sdist().requires + if "sdist" in self.builds or "external" in self.builds: + build_requires = self._frontend.get_requires_for_build_sdist().requires self.installer.install(build_requires, PythonPackageToxEnv.__name__, "requires_for_build_sdist") def _teardown(self) -> None: - if self._backend_executor_ is not None: + executor = self._frontend.backend_executor + if executor is not None: # pragma: no branch try: - if self._backend_executor.is_alive: - self._send("_exit") # try first on amicable shutdown + if executor.is_alive: + self._frontend._send("_exit") # try first on amicable shutdown except SystemExit: # pragma: no cover # if already has been interrupted ignore pass finally: - self._backend_executor_.close() + executor.close() super()._teardown() def perform_packaging(self, for_env: EnvConfigSet) -> List[Package]: """build the package to install""" of_type: str = for_env["package"] + + reqs: Optional[List[Requirement]] = None + if of_type == "wheel": + w_env = self._wheel_build_envs.get(for_env["wheel_build_env"]) + if w_env is not None and w_env is not self: + with w_env.display_context(self._has_display_suspended): + reqs = w_env.get_package_dependencies() if isinstance(w_env, Pep517VirtualEnvPackager) else [] + if reqs is None: + reqs = self.get_package_dependencies() + extras: Set[str] = for_env["extras"] - deps = self._dependencies_with_extras(self._get_package_dependencies(), extras) + deps = dependencies_with_extras(reqs, extras) if of_type == "dev-legacy": - deps = [*self.requires(), *self.get_requires_for_build_sdist().requires] + deps + deps = [*self.requires(), *self._frontend.get_requires_for_build_sdist().requires] + deps package: Package = DevLegacyPackage(self.core["tox_root"], deps) # the folder itself is the package elif of_type == "sdist": with self._pkg_lock: - package = SdistPackage(self.build_sdist(sdist_directory=self.pkg_dir).sdist, deps) + package = SdistPackage(self._frontend.build_sdist(sdist_directory=self.pkg_dir).sdist, deps) elif of_type == "wheel": - with self._pkg_lock: - path = self.build_wheel( - wheel_directory=self.pkg_dir, - metadata_directory=self.meta_folder, - config_settings=self._wheel_config_settings, - ).wheel - package = WheelPackage(path, deps) + w_env = self._wheel_build_envs.get(for_env["wheel_build_env"]) + if w_env is not None and w_env is not self: + with w_env.display_context(self._has_display_suspended): + return w_env.perform_packaging(for_env) + else: + with self._pkg_lock: + path = self._frontend.build_wheel( + wheel_directory=self.pkg_dir, + metadata_directory=self.meta_folder, + config_settings=self._wheel_config_settings, + ).wheel + package = WheelPackage(path, deps) else: # pragma: no cover # for when we introduce new packaging types and don't implement raise TypeError(f"cannot handle package type {of_type}") # pragma: no cover return [package] - def _get_package_dependencies(self) -> List[Requirement]: + def get_package_dependencies(self) -> List[Requirement]: with self._pkg_lock: if self._package_dependencies is None: # pragma: no branch self._ensure_meta_present() @@ -175,80 +188,59 @@ def _ensure_meta_present(self) -> None: if self._distribution_meta is not None: # pragma: no branch return # pragma: no cover self.setup() - dist_info = self.prepare_metadata_for_build_wheel(self.meta_folder, self._wheel_config_settings).metadata + dist_info = self._frontend.prepare_metadata_for_build_wheel( + self.meta_folder, self._wheel_config_settings + ).metadata self._distribution_meta = Distribution.at(str(dist_info)) # type: ignore[no-untyped-call] - @staticmethod - def _dependencies_with_extras(deps: List[Requirement], extras: Set[str]) -> List[Requirement]: - result: List[Requirement] = [] - for req in deps: - req = deepcopy(req) - markers: List[Union[str, Tuple[Variable, Variable, Variable]]] = getattr(req.marker, "_markers", []) or [] - # find the extra marker (if has) - _at: Optional[int] = None - extra: Optional[str] = None - for _at, (marker_key, op, marker_value) in ( - (_at_marker, marker) - for _at_marker, marker in enumerate(markers) - if isinstance(marker, tuple) and len(marker) == 3 - ): - if marker_key.value == "extra" and op.value == "==": # pragma: no branch - extra = marker_value.value - del markers[_at] - _at -= 1 - if _at > 0 and (isinstance(markers[_at], str) and markers[_at] in ("and", "or")): - del markers[_at] - if len(markers) == 0: - req.marker = None - break - if not (extra is None or extra in extras): - continue - result.append(req) - return result - - @contextmanager - def _wheel_directory(self) -> Iterator[Path]: - yield self.pkg_dir # use our local wheel directory for building wheel - @property def _wheel_config_settings(self) -> Optional[ConfigSettings]: return {"--global-option": ["--bdist-dir", str(self.env_dir / "build")]} - @property - def _backend_executor(self) -> LocalSubProcessPep517Executor: - if self._backend_executor_ is None: - self._backend_executor_ = LocalSubProcessPep517Executor( - colored=self.options.is_colored, - cmd=self.backend_cmd, - env=self._environment_variables, - cwd=self._root, - ) + def requires(self) -> Tuple[Requirement, ...]: + return self._frontend.requires - return self._backend_executor_ + +class Pep517VirtualEnvFrontend(Frontend): + def __init__(self, root: Path, env: Pep517VirtualEnvPackager) -> None: + super().__init__(*Frontend.create_args_from_folder(root)) + self._tox_env = env + self._backend_executor_: Optional[LocalSubProcessPep517Executor] = None + into: Dict[str, Any] = {} + pkg_cache = cached(into, key=lambda *args, **kwargs: "wheel" if "wheel_directory" in kwargs else "sdist") + self.build_wheel = pkg_cache(self.build_wheel) # type: ignore + self.build_sdist = pkg_cache(self.build_sdist) # type: ignore @property def backend_cmd(self) -> Sequence[str]: return ["python"] + self.backend_args - @property - def _environment_variables(self) -> Dict[str, str]: - env = super()._environment_variables - backend = os.pathsep.join(str(i) for i in self._backend_paths).strip() - if backend: - env["PYTHONPATH"] = backend - return env + def _send(self, cmd: str, **kwargs: Any) -> Tuple[Any, str, str]: + try: + if cmd == "prepare_metadata_for_build_wheel": + # given we'll build a wheel we might skip the prepare step + if "wheel" in self._tox_env.builds: + result = { + "code": 1, + "exc_type": "AvoidRedundant", + "exc_msg": "will need to build wheel either way, avoid prepare", + } + raise BackendFailed(result, "", "") + return super()._send(cmd, **kwargs) + except BackendFailed as exception: + raise exception if isinstance(exception, ToxBackendFailed) else ToxBackendFailed(exception) from exception @contextmanager def _send_msg( self, cmd: str, result_file: Path, msg: str # noqa: U100 ) -> Iterator[ToxCmdStatus]: # type: ignore[override] - with self.execute_async( + with self._tox_env.execute_async( cmd=self.backend_cmd, cwd=self._root, stdin=StdinSource.API, show=None, run_id=cmd, - executor=self._backend_executor, + executor=self.backend_executor, ) as execute_status: execute_status.write_stdin(f"{msg}{os.linesep}") yield ToxCmdStatus(execute_status) @@ -256,31 +248,33 @@ def _send_msg( if outcome is not None: # pragma: no branch outcome.assert_success() - def _send(self, cmd: str, **kwargs: Any) -> Tuple[Any, str, str]: - try: - if cmd == "prepare_metadata_for_build_wheel": - # given we'll build a wheel we might skip the prepare step - if "wheel" in self._builds: - result = { - "code": 1, - "exc_type": "AvoidRedundant", - "exc_msg": "will need to build wheel either way, avoid prepare", - } - raise BackendFailed(result, "", "") - return super()._send(cmd, **kwargs) - except BackendFailed as exception: - raise exception if isinstance(exception, ToxBackendFailed) else ToxBackendFailed(exception) from exception - def _unexpected_response(self, cmd: str, got: Any, expected_type: Any, out: str, err: str) -> NoReturn: try: super()._unexpected_response(cmd, got, expected_type, out, err) except BackendFailed as exception: raise exception if isinstance(exception, ToxBackendFailed) else ToxBackendFailed(exception) from exception - def requires(self) -> Tuple[Requirement, ...]: - return self._requires + @property + def backend_executor(self) -> LocalSubProcessPep517Executor: + if self._backend_executor_ is None: + environment_variables = self._tox_env.environment_variables.copy() + backend = os.pathsep.join(str(i) for i in self._backend_paths).strip() + if backend: + environment_variables["PYTHONPATH"] = backend + self._backend_executor_ = LocalSubProcessPep517Executor( + colored=self._tox_env.options.is_colored, + cmd=self.backend_cmd, + env=environment_variables, + cwd=self._root, + ) + + return self._backend_executor_ + + @contextmanager + def _wheel_directory(self) -> Iterator[Path]: + yield self._tox_env.pkg_dir # use our local wheel directory for building wheel @impl def tox_register_tox_env(register: ToxEnvRegister) -> None: - register.add_package_env(Pep517VirtualEnvPackage) + register.add_package_env(Pep517VirtualEnvPackager) diff --git a/src/tox/tox_env/python/virtual_env/package/util.py b/src/tox/tox_env/python/virtual_env/package/util.py new file mode 100644 index 000000000..2fe021957 --- /dev/null +++ b/src/tox/tox_env/python/virtual_env/package/util.py @@ -0,0 +1,33 @@ +from copy import deepcopy +from typing import List, Optional, Set, Tuple, Union + +from packaging.markers import Variable +from packaging.requirements import Requirement + + +def dependencies_with_extras(deps: List[Requirement], extras: Set[str]) -> List[Requirement]: + result: List[Requirement] = [] + for req in deps: + req = deepcopy(req) + markers: List[Union[str, Tuple[Variable, Variable, Variable]]] = getattr(req.marker, "_markers", []) or [] + # find the extra marker (if has) + _at: Optional[int] = None + extra: Optional[str] = None + for _at, (marker_key, op, marker_value) in ( + (_at_marker, marker) + for _at_marker, marker in enumerate(markers) + if isinstance(marker, tuple) and len(marker) == 3 + ): + if marker_key.value == "extra" and op.value == "==": # pragma: no branch + extra = marker_value.value + del markers[_at] + _at -= 1 + if _at > 0 and (isinstance(markers[_at], str) and markers[_at] in ("and", "or")): + del markers[_at] + if len(markers) == 0: + req.marker = None + break + if not (extra is None or extra in extras): + continue + result.append(req) + return result diff --git a/src/tox/tox_env/python/virtual_env/runner.py b/src/tox/tox_env/python/virtual_env/runner.py index 8104e91d4..0963ea385 100644 --- a/src/tox/tox_env/python/virtual_env/runner.py +++ b/src/tox/tox_env/python/virtual_env/runner.py @@ -18,16 +18,17 @@ def id() -> str: return "virtualenv" @property - def _default_package_tox_env_type(self) -> str: + def _package_tox_env_type(self) -> str: return "virtualenv-pep-517" + @property + def _external_pkg_tox_env_type(self) -> str: + return "virtualenv-cmd-builder" + @property def default_pkg_type(self) -> str: tox_root: Path = self.core["tox_root"] - if not ( - any((tox_root / i).exists() for i in ("pyproject.toml", "setup.py", "setup.cfg")) - or getattr(self.options, "install_pkg", None) is not None - ): + if not (any((tox_root / i).exists() for i in ("pyproject.toml", "setup.py", "setup.cfg"))): return "skip" return super().default_pkg_type diff --git a/src/tox/tox_env/runner.py b/src/tox/tox_env/runner.py index 55930648e..800b898b3 100644 --- a/src/tox/tox_env/runner.py +++ b/src/tox/tox_env/runner.py @@ -4,7 +4,7 @@ from abc import ABC, abstractmethod from hashlib import sha256 from pathlib import Path -from typing import Any, Dict, Iterable, Iterator, List, Tuple, cast +from typing import Any, Dict, Iterable, List, Optional, Tuple, cast from tox.config.types import Command, EnvList from tox.journal import EnvJournal @@ -15,7 +15,7 @@ class RunToxEnv(ToxEnv, ABC): def __init__(self, create_args: ToxEnvCreateArgs) -> None: - self._package_envs: Dict[str, PackageToxEnv] = {} + self.package_env: Optional[PackageToxEnv] = None self._packages: List[Package] = [] super().__init__(create_args) @@ -79,7 +79,9 @@ def ensure_one_line(value: str) -> str: default=False, desc="if set to true a failing result of this testenv will not make tox fail (instead just warn)", ) - if self._register_package_conf(): + has_external_pkg = getattr(self.options, "install_pkg", None) is not None + if self._register_package_conf() or has_external_pkg: + has_external_pkg = has_external_pkg or self.conf["package"] == "external" self.core.add_config( keys=["package_env", "isolated_build_env"], of_type=str, @@ -89,13 +91,14 @@ def ensure_one_line(value: str) -> str: self.conf.add_config( keys=["package_env"], of_type=str, - default=self.core["package_env"], + default=f'{self.core["package_env"]}{"_external" if has_external_pkg else ""}', desc="tox environment used to package", ) + is_external = self.conf["package"] == "external" self.conf.add_constant( keys=["package_tox_env_type"], - desc="tox package type used to package", - value=self._default_package_tox_env_type, + desc="tox package type used to generate the package", + value=self._external_pkg_tox_env_type if is_external else self._package_tox_env_type, ) def _teardown(self) -> None: @@ -106,14 +109,10 @@ def interrupt(self) -> None: super().interrupt() self._call_pkg_envs("interrupt") - def iter_package_env_types(self) -> Iterator[Tuple[str, str, str]]: - if "package_env" in self.conf: - name, pkg_env_type = self.conf["package_env"], self.conf["package_tox_env_type"] - yield "default", name, pkg_env_type - - def notify_of_package_env(self, tag: str, env: PackageToxEnv) -> None: - self._package_envs[tag] = env - env.notify_of_run_env(self.conf) + def get_package_env_types(self) -> Optional[Tuple[str, str]]: + if "package_env" not in self.conf: + return None + return self.conf["package_env"], self.conf["package_tox_env_type"] def _call_pkg_envs(self, method_name: str, *args: Any) -> None: for package_env in self.package_envs: @@ -131,11 +130,16 @@ def _default_package_env(self) -> str: @property @abstractmethod - def _default_package_tox_env_type(self) -> str: + def _package_tox_env_type(self) -> str: + raise NotImplementedError + + @property + @abstractmethod + def _external_pkg_tox_env_type(self) -> str: raise NotImplementedError def _setup_with_env(self) -> None: - if self._package_envs: + if self.package_env is not None: skip_pkg_install: bool = getattr(self.options, "skip_pkg_install", False) if skip_pkg_install is True: logging.warning("skip building and installing the package") @@ -186,9 +190,10 @@ def _handle_journal_package(journal: EnvJournal, packages: List[Package]) -> Non journal["installpkg"] = installed_meta[0] if len(installed_meta) == 1 else installed_meta @property - def _environment_variables(self) -> Dict[str, str]: - environment_variables = super()._environment_variables - if self._package_envs and self._packages: # if package(s) have been built insert them as environment variable + def environment_variables(self) -> Dict[str, str]: + environment_variables = super().environment_variables + if self.package_env is not None and self._packages: + # if package(s) have been built insert them as environment variable environment_variables["TOX_PACKAGE"] = os.pathsep.join(str(i) for i in self._packages) return environment_variables @@ -199,4 +204,6 @@ def _build_packages(self) -> List[Package]: @property def package_envs(self) -> Iterable[PackageToxEnv]: - yield from dict.fromkeys(self._package_envs.values()).keys() + if self.package_env is not None: + yield self.package_env + yield from self.package_env.child_pkg_envs(self.conf) diff --git a/src/tox/util/pep517/frontend.py b/src/tox/util/pep517/frontend.py index 146c72d80..3809ac200 100644 --- a/src/tox/util/pep517/frontend.py +++ b/src/tox/util/pep517/frontend.py @@ -98,7 +98,7 @@ def __init__( self._backend_paths = backend_paths self._backend_module = backend_module self._backend_obj = backend_obj - self._requires = requires + self.requires: Tuple[Requirement, ...] = requires self._reuse_backend = reuse_backend @classmethod diff --git a/tests/conftest.py b/tests/conftest.py index e1770ab7b..722ab49cd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -124,3 +124,21 @@ def fake_exe_on_path(tmp_path_factory: TempPathFactory) -> Iterator[Path]: maker.make(f"{cmd_name} = b:c") with patch.dict(os.environ, {"PATH": f"{tmp_path}{os.pathsep}{os.environ['PATH']}"}): yield tmp_path / cmd_name + + +@pytest.fixture(scope="session") +def demo_pkg_inline_wheel(tmp_path_factory: TempPathFactory, demo_pkg_inline: Path) -> Path: + return build_pkg(tmp_path_factory.mktemp("dist"), demo_pkg_inline, ["wheel"]) + + +def build_pkg(dist_dir: Path, of: Path, distributions: List[str], isolation: bool = True) -> Path: + from build.__main__ import build_package + + build_package(str(of), str(dist_dir), distributions=distributions, isolation=isolation) + package = next(dist_dir.iterdir()) + return package + + +@pytest.fixture(scope="session") +def pkg_builder() -> Callable[[Path, Path, List[str], bool], Path]: + return build_pkg diff --git a/tests/test_provision.py b/tests/test_provision.py index 2b631477d..226e723a3 100644 --- a/tests/test_provision.py +++ b/tests/test_provision.py @@ -5,7 +5,7 @@ from contextlib import contextmanager from pathlib import Path from subprocess import check_call -from typing import Iterator, List, Optional +from typing import Callable, Iterator, List, Optional from zipfile import ZipFile import pytest @@ -33,9 +33,11 @@ def elapsed(msg: str) -> Iterator[None]: @pytest.fixture(scope="session") -def tox_wheel(tmp_path_factory: TempPathFactory, worker_id: str) -> Path: +def tox_wheel( + tmp_path_factory: TempPathFactory, worker_id: str, pkg_builder: Callable[[Path, Path, List[str], bool], Path] +) -> Path: if worker_id == "master": # if not running under xdist we can just return - return _make_tox_wheel(tmp_path_factory) # pragma: no cover + return _make_tox_wheel(tmp_path_factory, pkg_builder) # pragma: no cover # otherwise we need to ensure only one worker creates the wheel, and the rest reuses root_tmp_dir = tmp_path_factory.getbasetemp().parent cache_file = root_tmp_dir / "tox_wheel.json" @@ -43,12 +45,14 @@ def tox_wheel(tmp_path_factory: TempPathFactory, worker_id: str) -> Path: if cache_file.is_file(): data = Path(json.loads(cache_file.read_text())) # pragma: no cover else: - data = _make_tox_wheel(tmp_path_factory) + data = _make_tox_wheel(tmp_path_factory, pkg_builder) cache_file.write_text(json.dumps(str(data))) return data -def _make_tox_wheel(tmp_path_factory: TempPathFactory) -> Path: +def _make_tox_wheel( + tmp_path_factory: TempPathFactory, pkg_builder: Callable[[Path, Path, List[str], bool], Path] +) -> Path: with elapsed("acquire current tox wheel"): # takes around 3.2s on build package: Optional[Path] = None if "TOX_PACKAGE" in os.environ: @@ -58,7 +62,7 @@ def _make_tox_wheel(tmp_path_factory: TempPathFactory) -> Path: if package is None: # when we don't get a wheel path injected, build it (for example when running from an IDE) into = tmp_path_factory.mktemp("dist") # pragma: no cover - package = build_wheel(into, Path(__file__).parents[1], isolation=False) # pragma: no cover + package = pkg_builder(into, Path(__file__).parents[1], ["wheel"], False) # pragma: no cover return package @@ -85,19 +89,6 @@ def tox_wheels(tox_wheel: Path, tmp_path_factory: TempPathFactory) -> List[Path] return result -@pytest.fixture(scope="session") -def demo_pkg_inline_wheel(tmp_path_factory: TempPathFactory, demo_pkg_inline: Path) -> Path: - return build_wheel(tmp_path_factory.mktemp("dist"), demo_pkg_inline) - - -def build_wheel(dist_dir: Path, of: Path, isolation: bool = True) -> Path: - from build.__main__ import build_package - - build_package(str(of), str(dist_dir), distributions=["wheel"], isolation=isolation) - package = next(dist_dir.iterdir()) - return package - - @pytest.fixture(scope="session") def pypi_index_self(pypi_server: IndexServer, tox_wheels: List[Path], demo_pkg_inline_wheel: Path) -> Index: with elapsed("start devpi and create index"): # takes around 1s diff --git a/tests/tox_env/python/pip/req/test_file.py b/tests/tox_env/python/pip/req/test_file.py index 16f1c1dd5..d66a0ca38 100644 --- a/tests/tox_env/python/pip/req/test_file.py +++ b/tests/tox_env/python/pip/req/test_file.py @@ -1,3 +1,4 @@ +import os import sys from contextlib import contextmanager from io import BytesIO @@ -84,7 +85,7 @@ pytest.param("--editable a", {}, ["-e a"], ["-e", "a"], id="editable"), pytest.param("--editable .[2,1]", {}, ["-e .[1,2]"], ["-e", ".[1,2]"], id="editable extra"), pytest.param(".[\t, a1. , B2-\t, C3_, ]", {}, [".[B2-,C3_,a1.]"], [".[B2-,C3_,a1.]"], id="path with extra"), - pytest.param(".[a.1]", {}, [".[a.1]"], [".[a.1]"], id="path with invalid extra is path"), + pytest.param(".[a.1]", {}, [f".{os.sep}.[a.1]"], [f".{os.sep}.[a.1]"], id="path with invalid extra is path"), pytest.param("-f a", {"find_links": ["a"]}, [], ["-f", "a"], id="f"), pytest.param("--find-links a", {"find_links": ["a"]}, [], ["-f", "a"], id="find-links"), pytest.param("--trusted-host a", {"trusted_hosts": ["a"]}, [], ["--trusted-host", "a"], id="trusted-host"), @@ -297,7 +298,7 @@ def test_requirements_env_var_missing(monkeypatch: MonkeyPatch, tmp_path: Path) req_file = RequirementsFile(requirements_file, constraint=False) assert vars(req_file.options) == {} found = [str(i) for i in req_file.requirements] - assert found == ["${ENV_VAR}"] + assert found == [f".{os.sep}${{ENV_VAR}}"] @pytest.mark.parametrize("flag", ["-r", "--requirement"]) diff --git a/tests/tox_env/python/pip/test_pip_install.py b/tests/tox_env/python/pip/test_pip_install.py index 0b11277f6..b1d4208d3 100644 --- a/tests/tox_env/python/pip/test_pip_install.py +++ b/tests/tox_env/python/pip/test_pip_install.py @@ -1,3 +1,4 @@ +import os from pathlib import Path from typing import Any, List from unittest.mock import Mock @@ -53,6 +54,15 @@ def test_pip_install_new_flag_recreates(tox_project: ToxProjectCreator) -> None: assert "install_deps> python -I -m pip install a -i i" in result_second.out +def test_pip_install_path(tox_project: ToxProjectCreator) -> None: + proj = tox_project({"tox.ini": "[testenv:py]\ndeps=.{/}a\nskip_install=true"}) + execute_calls = proj.patch_execute(lambda r: 0 if "install" in r.run_id else None) + + result = proj.run("r") + result.assert_success() + assert execute_calls.call_args[0][3].cmd == ["python", "-I", "-m", "pip", "install", f".{os.sep}a"] + + @pytest.mark.parametrize( ("content", "args"), [ diff --git a/tests/tox_env/python/virtual_env/package/__init__.py b/tests/tox_env/python/virtual_env/package/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/tox_env/python/virtual_env/package/conftest.py b/tests/tox_env/python/virtual_env/package/conftest.py new file mode 100644 index 000000000..9e673823b --- /dev/null +++ b/tests/tox_env/python/virtual_env/package/conftest.py @@ -0,0 +1,38 @@ +import sys +from pathlib import Path +from textwrap import dedent + +import pytest +from _pytest.tmpdir import TempPathFactory + + +@pytest.fixture(scope="session") +def pkg_with_extras_project(tmp_path_factory: TempPathFactory) -> Path: + py_ver = ".".join(str(i) for i in sys.version_info[0:2]) + setup_cfg = f""" + [metadata] + name = demo + version = 1.0.0 + [options] + packages = find: + install_requires = + platformdirs>=2.1 + colorama>=0.4.3 + + [options.extras_require] + testing = + covdefaults>=1.2; python_version == '2.7' or python_version == '{py_ver}' + pytest>=5.4.1; python_version == '{py_ver}' + docs = + sphinx>=3 + sphinx-rtd-theme>=0.4.3,<1 + format = + black>=3 + flake8 + """ + tmp_path = tmp_path_factory.mktemp("prj") + (tmp_path / "setup.cfg").write_text(dedent(setup_cfg)) + (tmp_path / "setup.py").write_text("from setuptools import setup; setup()") + toml = '[build-system]\nrequires=["setuptools", "wheel"]\nbuild-backend = "setuptools.build_meta"' + (tmp_path / "pyproject.toml").write_text(toml) + return tmp_path diff --git a/tests/tox_env/python/virtual_env/package/test_package_cmd_builder.py b/tests/tox_env/python/virtual_env/package/test_package_cmd_builder.py new file mode 100644 index 000000000..6a1b60652 --- /dev/null +++ b/tests/tox_env/python/virtual_env/package/test_package_cmd_builder.py @@ -0,0 +1,144 @@ +from pathlib import Path +from typing import Callable, List +from zipfile import ZipFile + +import pytest + +from tox.pytest import ToxProjectCreator + + +@pytest.fixture(scope="session") +def pkg_with_extras_project_wheel( + pkg_with_extras_project: Path, pkg_builder: Callable[[Path, Path, List[str], bool], Path] +) -> Path: + dist = pkg_with_extras_project / "dist" + pkg_builder(dist, pkg_with_extras_project, ["wheel"], False) + return next(dist.iterdir()) + + +def test_tox_install_pkg_wheel(tox_project: ToxProjectCreator, pkg_with_extras_project_wheel: Path) -> None: + proj = tox_project({"tox.ini": "[testenv]\nextras=docs,format"}) + execute_calls = proj.patch_execute(lambda r: 0 if "install" in r.run_id else None) + result = proj.run("r", "-e", "py", "--installpkg", str(pkg_with_extras_project_wheel)) + result.assert_success() + calls = [(i[0][0].conf.name, i[0][3].run_id, i[0][3].cmd[5:]) for i in execute_calls.call_args_list] + deps = ["black>=3", "colorama>=0.4.3", "flake8", "platformdirs>=2.1", "sphinx-rtd-theme<1,>=0.4.3", "sphinx>=3"] + expected = [ + ("py", "install_package_deps", deps), + ("py", "install_package", ["--force-reinstall", "--no-deps", str(pkg_with_extras_project_wheel)]), + ] + assert calls == expected + + +@pytest.fixture() +def pkg_with_extras_project_sdist( + pkg_with_extras_project: Path, pkg_builder: Callable[[Path, Path, List[str], bool], Path] +) -> Path: + dist = pkg_with_extras_project / "sdist" + pkg_builder(dist, pkg_with_extras_project, ["sdist"], False) + return next(dist.iterdir()) + + +def test_tox_install_pkg_sdist(tox_project: ToxProjectCreator, pkg_with_extras_project_sdist: Path) -> None: + proj = tox_project({"tox.ini": "[testenv]\nextras=docs,format"}) + execute_calls = proj.patch_execute(lambda r: 0 if "install" in r.run_id else None) + result = proj.run("r", "-e", "py", "--installpkg", str(pkg_with_extras_project_sdist)) + result.assert_success() + calls = [(i[0][0].conf.name, i[0][3].run_id, i[0][3].cmd[5:]) for i in execute_calls.call_args_list] + deps = ["black>=3", "colorama>=0.4.3", "flake8", "platformdirs>=2.1", "sphinx-rtd-theme<1,>=0.4.3", "sphinx>=3"] + assert calls == [ + (".pkg_external_sdist_meta", "install_requires", ["setuptools", "wheel"]), + (".pkg_external_sdist_meta", "get_requires_for_build_sdist", []), + (".pkg_external_sdist_meta", "prepare_metadata_for_build_wheel", []), + ("py", "install_package_deps", deps), + ("py", "install_package", ["--force-reinstall", "--no-deps", str(pkg_with_extras_project_sdist)]), + (".pkg_external_sdist_meta", "_exit", []), + ] + + +@pytest.mark.parametrize("mode", ["p", "le"]) # no need for r as is tested above +def test_install_pkg_via(tox_project: ToxProjectCreator, mode: str, pkg_with_extras_project_wheel: Path) -> None: + proj = tox_project({"tox.ini": "[testenv]\npackage=wheel"}) + execute_calls = proj.patch_execute(lambda r: 0 if "install" in r.run_id else None) + + result = proj.run(mode, "--installpkg", str(pkg_with_extras_project_wheel)) + + result.assert_success() + calls = [(i[0][0].conf.name, i[0][3].run_id) for i in execute_calls.call_args_list] + assert calls == [("py", "install_package_deps"), ("py", "install_package")] + + +@pytest.mark.usefixtures("enable_pip_pypi_access") +def test_build_wheel_external(tox_project: ToxProjectCreator, demo_pkg_inline: Path) -> None: + ini = """ + [testenv] + package = external + package_env = .ext + commands = + python -c 'from demo_pkg_inline import do; do()' + + [testenv:.ext] + deps = build + package_glob = {envtmpdir}{/}dist{/}*.whl + commands = + pyproject-build -w . -o {envtmpdir}{/}dist + """ + project = tox_project({"tox.ini": ini}) + result = project.run("r", "--root", str(demo_pkg_inline)) + + result.assert_success() + assert "greetings from demo_pkg_inline" in result.out + + +def test_build_wheel_external_fail_build(tox_project: ToxProjectCreator) -> None: + ini = """ + [testenv] + package = external + [testenv:.pkg_external] + commands = xyz + """ + project = tox_project({"tox.ini": ini}) + result = project.run("r") + + result.assert_failed() + assert "stopping as failed to build package" in result.out, result.out + + +def test_build_wheel_external_fail_no_pkg(tox_project: ToxProjectCreator) -> None: + ini = """ + [testenv] + package = external + """ + project = tox_project({"tox.ini": ini}) + result = project.run("r") + + result.assert_failed() + assert "failed with no package found in " in result.out, result.out + + +def test_build_wheel_external_fail_many_pkg(tox_project: ToxProjectCreator) -> None: + ini = """ + [testenv] + package = external + [testenv:.pkg_external] + commands = + python -c 'from pathlib import Path; (Path(r"{env_tmp_dir}") / "dist").mkdir()' + python -c 'from pathlib import Path; (Path(r"{env_tmp_dir}") / "dist" / "a").write_text("")' + python -c 'from pathlib import Path; (Path(r"{env_tmp_dir}") / "dist" / "b").write_text("")' + """ + project = tox_project({"tox.ini": ini}) + result = project.run("r") + + result.assert_failed() + assert "failed with found more than one package " in result.out, result.out + + +def test_tox_install_pkg_bad_wheel(tox_project: ToxProjectCreator, tmp_path: Path) -> None: + wheel = tmp_path / "w.whl" + with ZipFile(str(wheel), "w"): + pass + proj = tox_project({"tox.ini": "[testenv]"}) + result = proj.run("r", "-e", "py", "--installpkg", str(wheel)) + + result.assert_failed() + assert "failed with no .dist-info inside " in result.out, result.out diff --git a/tests/tox_env/python/virtual_env/package/test_package_pep517.py b/tests/tox_env/python/virtual_env/package/test_package_pep517.py new file mode 100644 index 000000000..f5d7e3c79 --- /dev/null +++ b/tests/tox_env/python/virtual_env/package/test_package_pep517.py @@ -0,0 +1,59 @@ +from pathlib import Path + +import pytest + +from tox.pytest import ToxProjectCreator + + +@pytest.mark.parametrize( + "pkg_type", + ["dev-legacy", "sdist", "wheel"], +) +def test_tox_ini_package_type_valid(tox_project: ToxProjectCreator, pkg_type: str) -> None: + proj = tox_project({"tox.ini": f"[testenv]\npackage={pkg_type}", "pyproject.toml": ""}) + result = proj.run("c", "-k", "package_tox_env_type") + result.assert_success() + res = result.env_conf("py")["package"] + assert res == pkg_type + got_type = result.env_conf("py")["package_tox_env_type"] + assert got_type == "virtualenv-pep-517" + + +def test_tox_ini_package_type_invalid(tox_project: ToxProjectCreator) -> None: + proj = tox_project({"tox.ini": "[testenv]\npackage=bad", "pyproject.toml": ""}) + result = proj.run("c", "-k", "package_tox_env_type") + result.assert_failed() + assert " invalid package config type bad requested, must be one of wheel, sdist, dev-legacy, skip" in result.out + + +def test_get_package_deps_different_extras(pkg_with_extras_project: Path, tox_project: ToxProjectCreator) -> None: + ini = "[testenv:a]\npackage=dev-legacy\nextras=docs\n[testenv:b]\npackage=sdist\nextras=format" + proj = tox_project({"tox.ini": ini}) + execute_calls = proj.patch_execute(lambda r: 0 if "install" in r.run_id else None) + result = proj.run("r", "--root", str(pkg_with_extras_project), "-e", "a,b") + result.assert_success() + installs = { + i[0][0].conf.name: i[0][3].cmd[5:] + for i in execute_calls.call_args_list + if i[0][3].run_id.startswith("install_package_deps") + } + assert installs == { + "a": ["colorama>=0.4.3", "platformdirs>=2.1", "setuptools", "sphinx-rtd-theme<1,>=0.4.3", "sphinx>=3", "wheel"], + "b": ["black>=3", "colorama>=0.4.3", "flake8", "platformdirs>=2.1"], + } + + +def test_package_root_via_root(tox_project: ToxProjectCreator, demo_pkg_inline: Path) -> None: + ini = f"[tox]\npackage_root={demo_pkg_inline}\n[testenv]\npackage=wheel\nwheel_build_env=.pkg" + proj = tox_project({"tox.ini": ini, "pyproject.toml": ""}) + proj.patch_execute(lambda r: 0 if "install" in r.run_id else None) + result = proj.run("r", "--notest") + result.assert_success() + + +def test_package_root_via_testenv(tox_project: ToxProjectCreator, demo_pkg_inline: Path) -> None: + ini = f"[testenv]\npackage=wheel\nwheel_build_env=.pkg\npackage_root={demo_pkg_inline}" + proj = tox_project({"tox.ini": ini, "pyproject.toml": ""}) + proj.patch_execute(lambda r: 0 if "install" in r.run_id else None) + result = proj.run("r", "--notest") + result.assert_success() diff --git a/tests/tox_env/python/virtual_env/package/test_python_package_util.py b/tests/tox_env/python/virtual_env/package/test_python_package_util.py new file mode 100644 index 000000000..062f381bd --- /dev/null +++ b/tests/tox_env/python/virtual_env/package/test_python_package_util.py @@ -0,0 +1,45 @@ +import sys +from itertools import zip_longest +from pathlib import Path + +import pytest +from packaging.requirements import Requirement + +from tox.tox_env.python.virtual_env.package.util import dependencies_with_extras +from tox.util.pep517.via_fresh_subprocess import SubprocessFrontend + +if sys.version_info >= (3, 8): # pragma: no cover (py38+) + from importlib.metadata import Distribution, PathDistribution # type: ignore[attr-defined] +else: # pragma: no cover ( PathDistribution: # type: ignore[no-any-unimported] + frontend = SubprocessFrontend(*SubprocessFrontend.create_args_from_folder(pkg_with_extras_project)[:-1]) + meta = pkg_with_extras_project / "meta" + result = frontend.prepare_metadata_for_build_wheel(meta) + return Distribution.at(result.metadata) + + +def test_load_dependency_no_extra(pkg_with_extras: PathDistribution) -> None: # type: ignore[no-any-unimported] + result = dependencies_with_extras([Requirement(i) for i in pkg_with_extras.requires], set()) + for left, right in zip_longest(result, (Requirement("platformdirs>=2.1"), Requirement("colorama>=0.4.3"))): + assert isinstance(right, Requirement) + assert str(left) == str(right) + + +def test_load_dependency_many_extra(pkg_with_extras: PathDistribution) -> None: # type: ignore[no-any-unimported] + py_ver = ".".join(str(i) for i in sys.version_info[0:2]) + result = dependencies_with_extras([Requirement(i) for i in pkg_with_extras.requires], {"docs", "testing"}) + exp = [ + Requirement("platformdirs>=2.1"), + Requirement("colorama>=0.4.3"), + Requirement("sphinx>=3"), + Requirement("sphinx-rtd-theme<1,>=0.4.3"), + Requirement(f'covdefaults>=1.2; python_version == "2.7" or python_version == "{py_ver}"'), + Requirement(f'pytest>=5.4.1; python_version == "{py_ver}"'), + ] + for left, right in zip_longest(result, exp): + assert isinstance(right, Requirement) + assert str(left) == str(right) diff --git a/tests/tox_env/python/virtual_env/test_package.py b/tests/tox_env/python/virtual_env/test_package.py deleted file mode 100644 index e46aca6a3..000000000 --- a/tests/tox_env/python/virtual_env/test_package.py +++ /dev/null @@ -1,136 +0,0 @@ -import sys -from itertools import zip_longest -from pathlib import Path -from textwrap import dedent - -import pytest -from packaging.requirements import Requirement - -from tox.pytest import TempPathFactory, ToxProjectCreator -from tox.tox_env.python.virtual_env.package.api import Pep517VirtualEnvPackage -from tox.util.pep517.via_fresh_subprocess import SubprocessFrontend - -if sys.version_info >= (3, 8): # pragma: no cover (py38+) - from importlib.metadata import Distribution, PathDistribution # type: ignore[attr-defined] -else: # pragma: no cover ( None: - proj = tox_project({"tox.ini": f"[testenv]\npackage={pkg_type}", "pyproject.toml": ""}) - result = proj.run("c", "-k", "package_tox_env_type") - result.assert_success() - res = result.env_conf("py")["package"] - assert res == pkg_type - got_type = result.env_conf("py")["package_tox_env_type"] - assert got_type == "virtualenv-pep-517" - - -def test_tox_ini_package_type_invalid(tox_project: ToxProjectCreator) -> None: - proj = tox_project({"tox.ini": "[testenv]\npackage=bad", "pyproject.toml": ""}) - result = proj.run("c", "-k", "package_tox_env_type") - result.assert_failed() - assert " invalid package config type bad requested, must be one of wheel, sdist, dev-legacy, skip" in result.out - - -@pytest.fixture(scope="session") -def pkg_with_extras_project(tmp_path_factory: TempPathFactory) -> Path: - py_ver = ".".join(str(i) for i in sys.version_info[0:2]) - setup_cfg = f""" - [metadata] - name = demo - [options] - packages = find: - install_requires = - platformdirs>=2.1 - colorama>=0.4.3 - - [options.extras_require] - testing = - covdefaults>=1.2; python_version == '2.7' or python_version == '{py_ver}' - pytest>=5.4.1; python_version == '{py_ver}' - docs = - sphinx>=3 - sphinx-rtd-theme>=0.4.3,<1 - format = - black>=3 - flake8 - """ - tmp_path = tmp_path_factory.mktemp("prj") - (tmp_path / "setup.cfg").write_text(dedent(setup_cfg)) - (tmp_path / "setup.py").write_text("from setuptools import setup; setup()") - toml = '[build-system]\nrequires=["setuptools", "wheel"]\nbuild-backend = "setuptools.build_meta"' - (tmp_path / "pyproject.toml").write_text(toml) - return tmp_path - - -@pytest.fixture(scope="session") -def pkg_with_extras(pkg_with_extras_project: Path) -> PathDistribution: # type: ignore[no-any-unimported] - frontend = SubprocessFrontend(*SubprocessFrontend.create_args_from_folder(pkg_with_extras_project)[:-1]) - meta = pkg_with_extras_project / "meta" - result = frontend.prepare_metadata_for_build_wheel(meta) - return Distribution.at(result.metadata) - - -def test_load_dependency_no_extra(pkg_with_extras: PathDistribution) -> None: # type: ignore[no-any-unimported] - result = Pep517VirtualEnvPackage._dependencies_with_extras( - [Requirement(i) for i in pkg_with_extras.requires], set() - ) - for left, right in zip_longest(result, (Requirement("platformdirs>=2.1"), Requirement("colorama>=0.4.3"))): - assert isinstance(right, Requirement) - assert str(left) == str(right) - - -def test_load_dependency_many_extra(pkg_with_extras: PathDistribution) -> None: # type: ignore[no-any-unimported] - py_ver = ".".join(str(i) for i in sys.version_info[0:2]) - result = Pep517VirtualEnvPackage._dependencies_with_extras( - [Requirement(i) for i in pkg_with_extras.requires], {"docs", "testing"} - ) - exp = [ - Requirement("platformdirs>=2.1"), - Requirement("colorama>=0.4.3"), - Requirement("sphinx>=3"), - Requirement("sphinx-rtd-theme<1,>=0.4.3"), - Requirement(f'covdefaults>=1.2; python_version == "2.7" or python_version == "{py_ver}"'), - Requirement(f'pytest>=5.4.1; python_version == "{py_ver}"'), - ] - for left, right in zip_longest(result, exp): - assert isinstance(right, Requirement) - assert str(left) == str(right) - - -def test_get_package_deps_different_extras(pkg_with_extras_project: Path, tox_project: ToxProjectCreator) -> None: - ini = "[testenv:a]\npackage=dev-legacy\nextras=docs\n[testenv:b]\npackage=sdist\nextras=format" - proj = tox_project({"tox.ini": ini}) - execute_calls = proj.patch_execute(lambda r: 0 if "install" in r.run_id else None) - result = proj.run("r", "--root", str(pkg_with_extras_project), "-e", "a,b") - result.assert_success() - installs = { - i[0][0].conf.name: i[0][3].cmd[5:] - for i in execute_calls.call_args_list - if i[0][3].run_id.startswith("install_package_deps") - } - assert installs == { - "a": ["colorama>=0.4.3", "platformdirs>=2.1", "setuptools", "sphinx-rtd-theme<1,>=0.4.3", "sphinx>=3", "wheel"], - "b": ["black>=3", "colorama>=0.4.3", "flake8", "platformdirs>=2.1"], - } - - -def test_package_root_via_root(tox_project: ToxProjectCreator, demo_pkg_inline: Path) -> None: - ini = f"[tox]\npackage_root={demo_pkg_inline}\n[testenv]\npackage=wheel\nwheel_build_env=.pkg" - proj = tox_project({"tox.ini": ini, "pyproject.toml": ""}) - proj.patch_execute(lambda r: 0 if "install" in r.run_id else None) - result = proj.run("r", "--notest") - result.assert_success() - - -def test_package_root_via_testenv(tox_project: ToxProjectCreator, demo_pkg_inline: Path) -> None: - ini = f"[testenv]\npackage=wheel\nwheel_build_env=.pkg\npackage_root={demo_pkg_inline}" - proj = tox_project({"tox.ini": ini, "pyproject.toml": ""}) - proj.patch_execute(lambda r: 0 if "install" in r.run_id else None) - result = proj.run("r", "--notest") - result.assert_success() diff --git a/tests/tox_env/python/virtual_env/test_setuptools.py b/tests/tox_env/python/virtual_env/test_setuptools.py index 1e5835a31..a6a8ac2f8 100644 --- a/tests/tox_env/python/virtual_env/test_setuptools.py +++ b/tests/tox_env/python/virtual_env/test_setuptools.py @@ -7,7 +7,7 @@ from tox.pytest import ToxProjectCreator from tox.tox_env.python.package import WheelPackage -from tox.tox_env.python.virtual_env.package.api import Pep517VirtualEnvPackage +from tox.tox_env.python.virtual_env.package.pep517 import Pep517VirtualEnvPackager @pytest.mark.integration() @@ -32,7 +32,7 @@ def test_setuptools_package( tox_env = outcome.state.tox_env("py") (package_env,) = list(tox_env.package_envs) - assert isinstance(package_env, Pep517VirtualEnvPackage) + assert isinstance(package_env, Pep517VirtualEnvPackager) packages = package_env.perform_packaging(tox_env.conf) assert len(packages) == 1 package = packages[0] diff --git a/tests/tox_env/python/virtual_env/test_virtualenv_api.py b/tests/tox_env/python/virtual_env/test_virtualenv_api.py index 490aa2954..67b11ddc3 100644 --- a/tests/tox_env/python/virtual_env/test_virtualenv_api.py +++ b/tests/tox_env/python/virtual_env/test_virtualenv_api.py @@ -156,21 +156,6 @@ def test_list_dependencies_command(tox_project: ToxProjectCreator) -> None: assert request.cmd == ["python", "-m", "pip", "freeze"] -@pytest.mark.parametrize("mode", ["r", "p", "le"]) -def test_install_pkg(tox_project: ToxProjectCreator, mode: str) -> None: - proj = tox_project({"tox.ini": "[testenv]\npackage=wheel"}) - execute_calls = proj.patch_execute(lambda r: 0 if "install" in r.run_id else None) - file = proj.path / "a" - file.write_text("") - - result = proj.run(mode, "--installpkg", str(file)) - - result.assert_success() - execute_calls.assert_called_once() - request: ExecuteRequest = execute_calls.call_args[0][3] - assert request.cmd == ["python", "-I", "-m", "pip", "install", str(file)] - - def test_can_build_and_run_python_2(tox_project: ToxProjectCreator, demo_pkg_inline: Path) -> None: try: session_via_cli(["-p", "2.7", "venv"]) diff --git a/whitelist.txt b/whitelist.txt index bf2808553..675749991 100644 --- a/whitelist.txt +++ b/whitelist.txt @@ -38,7 +38,6 @@ contnode copytree cov cpus -creationflags creq crypto ctrl @@ -229,6 +228,7 @@ unregister untyped url2pathname usedevelop +usefixtures util utils v3