Support for external packages and builders (#2235)

.github/workflows/check.yml

@@ -24,7 +24,7 @@ jobs:
           - Windows
           - MacOs
         py:
-          - "3.10.0-rc.1"
+          - "3.10.0-rc.2"
           - "3.9"
           - "3.8"
           - "3.7"

docs/changelog/2204.feature.rst

@@ -0,0 +1,2 @@
+Add ``external`` package type for :ref:`package` (see :ref:`external-package-builder`), and extract package dependencies
+for packages passed in via :ref:`--installpkg <tox-run---installpkg>` - by :user:`gaborbernat`.

docs/config.rst

@@ -476,6 +476,7 @@ Package
 
    Indicates where the packaging root file exists (historically setup.py file or pyproject.toml now).
 
+.. _python-options:
 
 Python options
 ~~~~~~~~~~~~~~
@@ -542,8 +543,8 @@ Python run
    :keys: package
    :version_added: 4.0
 
-   When option can be one of ``skip``, ``dev-legacy``, ``sdist`` or ``wheel``. If :ref:`use_develop` is set this becomes
-   a constant of ``dev-legacy``. If :ref:`skip_install` is set this becomes a constant of ``skip``.
+   When option can be one of ``skip``, ``dev-legacy``, ``sdist``, ``wheel`` or ``external``. If :ref:`use_develop` is
+   set this becomes a constant of ``dev-legacy``. If :ref:`skip_install` is set this becomes a constant of ``skip``.
 
 
 .. conf::
@@ -565,6 +566,59 @@ Python run
    A list of "extras" from the package to be installed. For example, ``extras = testing`` is equivalent to ``[testing]``
    in a ``pip install`` command.
 
+.. _external-package-builder:
+
+External package builder
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+tox supports operating with externally built packages. External packages might be provided in two wayas:
+
+- explicitly via the :ref:`--installpkg <tox-run---installpkg>` CLI argument,
+- setting the :ref:`package` to ``external`` and using a tox packaging environment named ``<package_env>_external``
+  (see :ref:`package_env`) to build the package. The tox packaging environment takes all configuration flags of a
+  :ref:`python environment <python-options>`, plus the following:
+
+.. conf::
+   :keys: deps
+   :default: <empty list>
+   :ref_suffix: external
+
+   Name of the Python dependencies as specified by `PEP-440`_. Installed into the environment prior running the build
+   commands. All installer commands are executed using the :ref:`tox_root` as the current working directory.
+
+.. conf::
+   :keys: commands
+   :default: <empty list>
+   :ref_suffix: external
+
+   Commands to run that will build the package. If any command fails the packaging operation is considered failed and
+   will fail all environments using that package.
+
+.. conf::
+   :keys: ignore_errors
+   :default: False
+   :ref_suffix: external
+
+   When executing the commands keep going even if a sub-command exits with non-zero exit code. The overall status will
+   be "commands failed", i.e. tox will exit non-zero in case any command failed. It may be helpful to note that this
+   setting is analogous to the ``-k`` or ``--keep-going`` option of GNU Make.
+
+.. conf::
+   :keys: change_dir, changedir
+   :default: {tox root}
+   :ref_suffix: external
+
+   Change to this working directory when executing the package build command. If the directory does not exist yet, it
+   will be created (required for Windows to be able to execute any command).
+
+.. conf::
+   :keys: package_glob
+   :default: {envtmpdir}{/}dist{/}*
+
+   A glob that should match the wheel/sdist file to install. If no file or multiple files is matched the packaging
+   operation is considered failed and will raise an error.
+
+
 Python virtual environment
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 .. conf::

src/tox/execute/pep517_backend.py

@@ -52,7 +52,7 @@ def local_execute(self, options: ExecuteOptions) -> Tuple[LocalSubProcessExecute
                     self.is_alive = True
                     break
                 if b"failed to start backend" in status.err:
-                    from tox.tox_env.python.virtual_env.package.api import ToxBackendFailed
+                    from tox.tox_env.python.virtual_env.package.pep517 import ToxBackendFailed
 
                     failure = BackendFailed(
                         result={

src/tox/plugin/manager.py

@@ -14,7 +14,7 @@
 from tox.session.cmd.run import parallel, sequential
 from tox.tox_env import package as package_api
 from tox.tox_env.python.virtual_env import runner
-from tox.tox_env.python.virtual_env.package import api
+from tox.tox_env.python.virtual_env.package import cmd_builder, pep517
 from tox.tox_env.register import REGISTER, ToxEnvRegister
 
 from ..config.main import Config
@@ -33,7 +33,8 @@ def __init__(self) -> None:
             loader_api,
             provision,
             runner,
-            api,
+            pep517,
+            cmd_builder,
             legacy,
             version_flag,
             exec_,

src/tox/pytest.py

@@ -301,19 +301,19 @@ def enable_pep517_backend_coverage() -> Iterator[None]:  # noqa: PT004
         yield  # pragma: no cover
         return  # pragma: no cover
     # the COV_ env variables needs to be passed on for the PEP-517 backend
-    from tox.tox_env.python.virtual_env.package.api import Pep517VirtualEnvPackage
+    from tox.tox_env.python.virtual_env.package.pep517 import Pep517VirtualEnvPackager
 
-    def default_pass_env(self: Pep517VirtualEnvPackage) -> List[str]:
+    def default_pass_env(self: Pep517VirtualEnvPackager) -> List[str]:
         result = previous(self)
         result.append("COV_*")
         return result
 
-    previous = Pep517VirtualEnvPackage._default_pass_env
+    previous = Pep517VirtualEnvPackager._default_pass_env
     try:
-        Pep517VirtualEnvPackage._default_pass_env = default_pass_env  # type: ignore
+        Pep517VirtualEnvPackager._default_pass_env = default_pass_env  # type: ignore
         yield
     finally:
-        Pep517VirtualEnvPackage._default_pass_env = previous  # type: ignore
+        Pep517VirtualEnvPackager._default_pass_env = previous  # type: ignore
 
 
 class ToxRunOutcome:

src/tox/session/cmd/run/single.py

@@ -8,8 +8,9 @@
 
 from tox.config.types import Command
 from tox.execute.api import Outcome, StdinSource
+from tox.tox_env.api import ToxEnv
 from tox.tox_env.errors import Fail, Skip
-from tox.tox_env.python.virtual_env.package.api import ToxBackendFailed
+from tox.tox_env.python.virtual_env.package.pep517 import ToxBackendFailed
 from tox.tox_env.runner import RunToxEnv
 
 LOGGER = logging.getLogger(__name__)
@@ -86,7 +87,7 @@ def run_commands(tox_env: RunToxEnv, no_test: bool) -> Tuple[int, List[Outcome]]
     return exit_code, outcomes
 
 
-def run_command_set(tox_env: RunToxEnv, key: str, cwd: Path, ignore_errors: bool, outcomes: List[Outcome]) -> int:
+def run_command_set(tox_env: ToxEnv, key: str, cwd: Path, ignore_errors: bool, outcomes: List[Outcome]) -> int:
     exit_code = Outcome.OK
     command_set: List[Command] = tox_env.conf[key]
     for at, cmd in enumerate(command_set):
@@ -113,5 +114,6 @@ def run_command_set(tox_env: RunToxEnv, key: str, cwd: Path, ignore_errors: bool
 
 __all__ = (
     "run_one",
+    "run_command_set",
     "ToxEnvRunResult",
 )

src/tox/session/state.py

@@ -79,10 +79,24 @@ def _build_run_env(self, env_conf: EnvConfigSet) -> None:
         self._build_package_env(env)
 
     def _build_package_env(self, env: RunToxEnv) -> None:
-        for tag, name, core_type in env.iter_package_env_types():
-            with self.log_handler.with_context(name):
-                package_tox_env = self._get_package_env(core_type, name)
-                env.notify_of_package_env(tag, package_tox_env)
+        pkg_info = env.get_package_env_types()
+        if pkg_info is not None:
+            name, core_type = pkg_info
+            env.package_env = self._build_pkg_env(name, core_type, env)
+
+    def _build_pkg_env(self, name: str, core_type: str, env: RunToxEnv) -> PackageToxEnv:
+        with self.log_handler.with_context(name):
+            package_tox_env = self._get_package_env(core_type, name)
+
+            child_package_envs = package_tox_env.register_run_env(env)
+            try:
+                child_name, child_type = next(child_package_envs)
+                while True:
+                    child_pkg_env = self._build_pkg_env(child_name, child_type, env)
+                    child_name, child_type = child_package_envs.send(child_pkg_env)
+            except StopIteration:
+                pass
+            return package_tox_env
 
     def _get_package_env(self, packager: str, name: str) -> PackageToxEnv:
         if name in self._pkg_env:  # if already created reuse

src/tox/tox_env/api.py

@@ -284,7 +284,7 @@ def _clean(self, transitive: bool = False) -> None:  # noqa: U100
         self._run_state.update({"setup": False, "clean": True})
 
     @property
-    def _environment_variables(self) -> Dict[str, str]:
+    def environment_variables(self) -> Dict[str, str]:
         pass_env: List[str] = self.conf["pass_env"]
         set_env: SetEnv = self.conf["set_env"]
         if self._env_vars_pass_env == pass_env and not set_env.changed and self._env_vars is not None:
@@ -372,7 +372,7 @@ def execute_async(
             cwd = self.core["tox_root"]
         if show is None:
             show = self.options.verbosity > 3
-        request = ExecuteRequest(cmd, cwd, self._environment_variables, stdin, run_id, allow=self._allow_externals)
+        request = ExecuteRequest(cmd, cwd, self.environment_variables, stdin, run_id, allow=self._allow_externals)
         if _CWD == request.cwd:
             repr_cwd = ""
         else:

src/tox/tox_env/package.py

@@ -4,13 +4,16 @@
 from abc import ABC, abstractmethod
 from pathlib import Path
 from threading import Lock
-from typing import List, Optional, Set, cast
+from typing import TYPE_CHECKING, Generator, Iterator, List, Optional, Set, Tuple, cast
 
 from tox.config.main import Config
 from tox.config.sets import EnvConfigSet
 
 from .api import ToxEnv, ToxEnvCreateArgs
 
+if TYPE_CHECKING:
+    from .runner import RunToxEnv
+
 
 class Package:
     """package"""
@@ -53,13 +56,19 @@ def _recreate_default(self, conf: "Config", value: Optional[str]) -> bool:
     def perform_packaging(self, for_env: EnvConfigSet) -> List[Package]:
         raise NotImplementedError
 
-    def notify_of_run_env(self, conf: EnvConfigSet) -> None:
-        with self._lock:
-            self._envs.add(conf.name)
-
     def teardown_env(self, conf: EnvConfigSet) -> None:
         with self._lock:
             self._envs.remove(conf.name)
             has_envs = bool(self._envs)
         if not has_envs:
             self._teardown()
+
+    def register_run_env(self, run_env: "RunToxEnv") -> Generator[Tuple[str, str], "PackageToxEnv", None]:
+        with self._lock:
+            self._envs.add(run_env.conf.name)
+        return
+        yield  # make this a generator
+
+    @abstractmethod
+    def child_pkg_envs(self, run_conf: EnvConfigSet) -> Iterator["PackageToxEnv"]:
+        raise NotImplementedError

src/tox/tox_env/python/package.py

@@ -3,12 +3,19 @@
 """
 from abc import ABC, abstractmethod
 from pathlib import Path
-from typing import Any, Sequence, Tuple
+from typing import TYPE_CHECKING, Any, Dict, Generator, Iterator, Optional, Sequence, Tuple, Union, cast
 
 from packaging.requirements import Requirement
 
+from ...config.sets import EnvConfigSet
+from ..api import ToxEnvCreateArgs
 from ..package import Package, PackageToxEnv, PathPackage
+from ..runner import RunToxEnv
 from .api import Python
+from .pip.req_file import PythonDeps
+
+if TYPE_CHECKING:
+    from tox.config.main import Config
 
 
 class PythonPackage(Package):
@@ -34,6 +41,10 @@ class DevLegacyPackage(PythonPathPackageWithDeps):
 
 
 class PythonPackageToxEnv(Python, PackageToxEnv, ABC):
+    def __init__(self, create_args: ToxEnvCreateArgs) -> None:
+        self._wheel_build_envs: Dict[str, PythonPackageToxEnv] = {}
+        super().__init__(create_args)
+
     def register_config(self) -> None:
         super().register_config()
 
@@ -43,5 +54,47 @@ def _setup_env(self) -> None:
         self.installer.install(self.requires(), PythonPackageToxEnv.__name__, "requires")
 
     @abstractmethod
-    def requires(self) -> Tuple[Requirement, ...]:
+    def requires(self) -> Union[Tuple[Requirement, ...], PythonDeps]:
         raise NotImplementedError
+
+    def register_run_env(self, run_env: RunToxEnv) -> Generator[Tuple[str, str], PackageToxEnv, None]:
+        yield from super().register_run_env(run_env)
+        if not isinstance(run_env, Python) or run_env.conf["package"] != "wheel" or "wheel_build_env" in run_env.conf:
+            return
+
+        def default_wheel_tag(conf: "Config", env_name: Optional[str]) -> str:
+            # https://www.python.org/dev/peps/pep-0427/#file-name-convention
+            # when building wheels we need to ensure that the built package is compatible with the target env
+            # compatibility is documented within https://www.python.org/dev/peps/pep-0427/#file-name-convention
+            # a wheel tag example: {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl
+            # python only code are often compatible at major level (unless universal wheel in which case both 2/3)
+            # c-extension codes are trickier, but as of today both poetry/setuptools uses pypa/wheels logic
+            # https://github.com/pypa/wheel/blob/master/src/wheel/bdist_wheel.py#L234-L280
+            run_py = cast(Python, run_env).base_python
+            if run_py is None:
+                raise ValueError(f"could not resolve base python for {self.conf.name}")
+
+            default_pkg_py = self.base_python
+            if (
+                default_pkg_py.version_no_dot == run_py.version_no_dot
+                and default_pkg_py.impl_lower == run_py.impl_lower
+            ):
+                return self.conf.name
+
+            return f"{self.conf.name}-{run_py.impl_lower}{run_py.version_no_dot}"
+
+        run_env.conf.add_config(
+            keys=["wheel_build_env"],
+            of_type=str,
+            default=default_wheel_tag,
+            desc="wheel tag to use for building applications",
+        )
+        pkg_env = run_env.conf["wheel_build_env"]
+        result = yield pkg_env, run_env.conf["package_tox_env_type"]
+        self._wheel_build_envs[pkg_env] = cast(PythonPackageToxEnv, result)
+
+    def child_pkg_envs(self, run_conf: EnvConfigSet) -> Iterator[PackageToxEnv]:
+        if run_conf["package"] == "wheel":
+            env = self._wheel_build_envs.get(run_conf["wheel_build_env"])
+            if env is not None and env.name != self.name:
+                yield env

src/tox/tox_env/python/pip/pip_install.py

@@ -135,8 +135,6 @@ def _install_list_of_deps(
             elif isinstance(arg, DevLegacyPackage):
                 groups["req"].extend(str(i) for i in arg.deps)
                 groups["dev_pkg"].append(str(arg.path))
-            elif isinstance(arg, PathPackage):
-                groups["path_pkg"].append(str(arg.path))
             else:
                 logging.warning(f"pip cannot install {arg!r}")
                 raise SystemExit(1)
@@ -158,8 +156,6 @@ def _install_list_of_deps(
             for entry in groups["dev_pkg"]:
                 install_args.extend(("-e", str(entry)))
             self._execute_installer(install_args, of_type)
-        if groups["path_pkg"]:
-            self._execute_installer(groups["path_pkg"], of_type)
 
     def _execute_installer(self, deps: Sequence[Any], of_type: str) -> None:
         cmd = self.build_install_cmd(deps)

src/tox/tox_env/python/pip/req/file.py

@@ -56,7 +56,9 @@ def __init__(self, req: str, options: Dict[str, Any], from_file: str, lineno: in
                 else:
                     path = root / req
                 extra_part = f"[{','.join(sorted(extras))}]" if extras else ""
-                rel_path = path.resolve().relative_to(root)
+                rel_path = str(path.resolve().relative_to(root))
+                if rel_path != "." and os.sep not in rel_path:  # prefix paths in cwd to not convert them to requirement
+                    rel_path = f".{os.sep}{rel_path}"
                 self._requirement = f"{rel_path}{extra_part}"
         self._options = options
         self._from_file = from_file

src/tox/tox_env/python/pip/req_file.py

@@ -77,6 +77,12 @@ def unroll(self) -> Tuple[List[str], List[str]]:
             self._unroll = result_opts, result_req
         return self._unroll
 
+    @classmethod
+    def factory(cls, root: Path, raw: object) -> "PythonDeps":
+        if not isinstance(raw, str):
+            raise TypeError(raw)
+        return cls(raw, root)
+
 
 ONE_ARG = {
     "-i",