From bea9e30c13d53f6f0dd0a2c5a85f8c0fab705a66 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sat, 14 Sep 2024 01:01:47 -0700 Subject: [PATCH] wip: better docs on how to define toolchains --- docs/index.md | 2 +- docs/toolchains.md | 200 ++++++++++++++++++++- sphinxdocs/inventories/bazel_inventory.txt | 9 +- sphinxdocs/src/sphinx_bzl/bzl.py | 18 +- 4 files changed, 220 insertions(+), 9 deletions(-) diff --git a/docs/index.md b/docs/index.md index 445cf20268..c06c31ed44 100644 --- a/docs/index.md +++ b/docs/index.md @@ -57,7 +57,7 @@ by buildifier. self getting-started pypi-dependencies -toolchains +Toolchains pip coverage precompiling diff --git a/docs/toolchains.md b/docs/toolchains.md index fac1bfc6b0..b5f664fcc3 100644 --- a/docs/toolchains.md +++ b/docs/toolchains.md @@ -161,9 +161,13 @@ Remember to call `use_repo()` to make repos visible to your module: #### Toolchain usage in other rules -Python toolchains can be utilized in other bazel rules, such as `genrule()`, by adding the `toolchains=["@rules_python//python:current_py_toolchain"]` attribute. You can obtain the path to the Python interpreter using the `$(PYTHON2)` and `$(PYTHON3)` ["Make" Variables](https://bazel.build/reference/be/make-variables). See the -{gh-path}`test_current_py_toolchain ` target for an example. - +Python toolchains can be utilized in other bazel rules, such as `genrule()`, by +adding the `toolchains=["@rules_python//python:current_py_toolchain"]` +attribute. You can obtain the path to the Python interpreter using the +`$(PYTHON2)` and `$(PYTHON3)` ["Make" +Variables](https://bazel.build/reference/be/make-variables). See the +{gh-path}`test_current_py_toolchain ` target +for an example. ## Workspace configuration @@ -242,3 +246,193 @@ there is a toolchain misconfiguration somewhere. To aid migration off the Bazel-builtin toolchain, rules_python provides {obj}`@rules_python//python/runtime_env_toolchains:all`. This is an equivalent toolchain, but is implemented using rules_python's objects. + + +## Custom toolchains + +While rules_python provides toolchains by default, it is not required to use +them, and you can define your own toolchains to use instead. This section +gives an introduction for how to define them yourself. + +:::{note} +* Defining your own toolchains is an advanced feature. +* APIs used for defining them are less stable and may change more often. +::: + +Under the hood, there are multiple toolchains that comprise the different +information necessary to build Python targets. Each one has an +associated _toolchain type_ that identifies it. We call the collection of these +toolchains a "toolchain suite". + +One of the underlying design goals of the toolchains is to support complex and +bespoke environments. Such environments may use an arbitrary combination of +{obj}`RBE`, cross-platform building, multiple Python versions, +building Python from source, embeding Python (as opposed to building separate +interpreters), using prebuilt binaries, or using binaries built from source. To +that end, many of the attributes they accept, and fields they provide, are +optional. + +### Target toolchain type + +The target toolchain type is {obj}`//python:toolchain_type`, and it +is for _target configuration_ runtime information, e.g., the Python version +and interpreter binary that a program will use. + +The is typically implemented using {obj}`py_runtime()`, which +provides the {obj}`PyRuntimeInfo` provider. For historical reasons from the +Python 2 transition, `py_runtime` is wrapped in {obj}`py_runtime_pair`, +which provides {obj}`ToolchainInfo` with the field `py3_runtime`, which is an +instance of `PyRuntimeInfo`. + +This toolchain type is intended to hold only _target configuration_ values. As +such, when defining its associated {external:bzl:obj}`toolchain` target, only +set {external:bzl:obj}`toolchain.target_compatible_with` and/or +{external:bzl:obj}`toolchain.target_settings` constraints; there is no need to +set {external:bzl:obj}`toolchain.exec_compatible_with`. + +### Python C toolchain type + +The Python C toolchain type ("py cc") is {obj}`//python/cc:toolchain_type`, and +it has C/C++ information for the _target configuration_, e.g. the C headers that +provide `Python.h`. + +This is typically implemented using {obj}`py_cc_toolchain()`, which provides +{obj}`ToolchainInfo` with the field `py_cc_toolchain` set, which is a +{obj}`PyCcToolchainInfo` provider instance. + +This toolchain type is intended to hold only _target configuration_ values +relating to the C/C++ information for the Python runtime. As such, when defining +its associated {external:obj}`toolchain` target, only set +{external:bzl:obj}`toolchain.target_compatible_with` and/or +{external:bzl:obj}`toolchain.target_settings` constraints; there is no need to +set {external:bzl:obj}`toolchain.exec_compatible_with`. + +### Exec tools toolchain type + +The exec tools toolchain type is {obj}`//python:exec_tools_toolchain_type`, +and it is for supporting tools for _building_ programs, e.g. the binary to +precompile code at build time. + +This toolchain type is intended to hold only _exec configuration_ values -- +usually tools (prebuilt or from-source) used to build Python targets. + +This is typically implemented using {obj}`py_exec_tools_toolchain`, which +provides {obj}`ToolchainInfo` with the field `exec_tools` set, which is an +instance of {obj}`PyExecToolsInfo`. + +The toolchain constraints of this toolchain type can be a bit more nuanced than +the other toolchain types. Typically, you set +{external:bzl:obj}`toolchain.target_settings` to the Python version the tools +are for, and {external:bzl:obj}`toolchain.exec_compatible_with` to the platform +they can run on. This allows the toolchain to first be considered based on the +target configuration (e.g. Python version), then for one to be chosen based on +finding one compatible with the available host platforms to run the tool on. + +However, what `target_compatible_with`/`target_settings` and +`exec_compatible_with` values to use depend on details of the tools being used. +For example: +* If you had a precompiler that supported any version of Python, then + putting the Python version in `target_settings` is unnecessary. +* If you had a prebuilt polyglot precompiler binary that could run on any + platform, then setting `exec_compatible_with` is unnecessary. + +This can work because, when the rules invoke these build tools, they pass along +all necessary information so that the tool can be entirely independent of the +target configuration being built for. + +Alternatively, if you had a precompiler that only ran on linux, and only +produced valid output for programs intended to run on linux, then _both_ +`exec_compatible_with` and `target_compatible_with` must be set to linux. + +### Custom toolchain example + +Here, we show an example for a semi-complicated toolchain suite, one that is: + +* A CPython-based interpreter +* For Python version 3.12.0 +* Using an in-build interpreter built from source +* That only runs on Linux +* Using a prebuilt precompiler that only runs on Linux, and only produces byte + code valid for 3.12 +* With the exec tools interpreter disabled (unnecessary with a prebuild + precompiler) +* Providing C headers and libraries + +Defining toolchains for this might look something like this: + +``` +# File: toolchain_impls/BUILD +load("@rules_python//python:py_cc_toolchain.bzl", "py_cc_toolchain") +load("@rules_python//python:py_exec_tools_toolchain.bzl", "py_exec_tools_toolchain") +load("@rules_python//python:py_runtime.bzl", "py_runtime") +load("@rules_python//python:py_runtime_pair.bzl", "py_runtime_pair") + +MAJOR = 3 +MINOR = 12 +MICRO = 0 + +py_runtime( + name = "runtime", + interpreter = ":python", + interpreter_version_info = { + "major": str(MAJOR), + "minor": str(MINOR), + "micro": str(MICRO), + } + implementation = "cpython" +) +py_runtime_pair( + name = "runtime_pair", + py3_runtime = ":runtime" +) + +py_cc_toolchain( + name = "py_cc_toolchain_impl", + headers = ":headers", + libs = ":libs", + python_version = "{}.{}".format(MAJOR, MINOR) +) + +py_exec_tools_toolchain( + name = "exec_tools_toolchain_impl", + exec_interpreter = "@rules_python/python:null_target", + precompiler = "precompiler-cpython-3.12" +) + +cc_binary(name = "python3.12", ...) +cc_library(name = "headers", ...) +cc_library(name = "libs", ...) + +# File: toolchains/BUILD +# Putting toolchain() calls in a separate package from the toolchain +# implementations minimizes Bazel loading overhead + +toolchain( + name = "runtime_toolchain", + toolchain = "//toolchain_impl:runtime_pair", + toolchain_type = "@rules_python//python:toolchain_type", + target_compatible_with = ["@platforms/os:linux"] +) +toolchain( + name = "py_cc_toolchain", + toolchain = "//toolchain_impl:py_cc_toolchain_impl", + toolchain_type = "@rules_python//python/cc:toolchain_type", + target_compatible_with = ["@platforms/os:linux"] +) + +toolchain( + name = "exec_tools_toolchain", + toolchain = "//toolchain_impl:exec_tools_toolchain_impl", + toolchain_type = "@rules_python//python:exec_tools_toolchain_type", + target_settings = [ + "@rules_python//python/config_settings:is_python_3.12", + ], + exec_comaptible_with = ["@platforms/os:linux"] +) +``` + +:::{note} +The toolchain() calls should be in a separate BUILD file from everything else. +This avoids Bazel having to perform unnecessary work when it discovers the list +of available toolchains. +::: diff --git a/sphinxdocs/inventories/bazel_inventory.txt b/sphinxdocs/inventories/bazel_inventory.txt index ee507d1ccc..fd2bac7062 100644 --- a/sphinxdocs/inventories/bazel_inventory.txt +++ b/sphinxdocs/inventories/bazel_inventory.txt @@ -9,6 +9,7 @@ ExecutionInfo bzl:type 1 rules/lib/providers/ExecutionInfo - File bzl:type 1 rules/lib/File - Label bzl:type 1 rules/lib/Label - Name bzl:type 1 concepts/labels#target-names - +RBE bzl:obj 1 remote/rbe - RunEnvironmentInfo bzl:type 1 rules/lib/providers/RunEnvironmentInfo - Target bzl:type 1 rules/lib/builtins/Target - ToolchainInfo bzl:type 1 rules/lib/providers/ToolchainInfo.html - @@ -58,6 +59,7 @@ ctx.version_file bzl:obj 1 rules/lib/builtins/ctx#version_file - ctx.workspace_name bzl:obj 1 rules/lib/builtins/ctx#workspace_name - depset bzl:type 1 rules/lib/depset - dict bzl:type 1 rules/lib/dict - +exec_compatible_with bzl:attribute 1 reference/be/common-definitions#common.exec_compatible_with - int bzl:type 1 rules/lib/int - label bzl:type 1 concepts/labels - list bzl:type 1 rules/lib/list - @@ -131,8 +133,13 @@ runfiles.root_symlinks bzl:type 1 rules/lib/builtins/runfiles#root_symlinks - runfiles.symlinks bzl:type 1 rules/lib/builtins/runfiles#symlinks - str bzl:type 1 rules/lib/string - struct bzl:type 1 rules/lib/builtins/struct - +target_compatible_with bzl:attribute 1 reference/be/common-definitions#common.target_compatible_with - testing bzl:obj 1 rules/lib/toplevel/testing - testing.ExecutionInfo bzl:function 1 rules/lib/toplevel/testing#ExecutionInfo - testing.TestEnvironment bzl:function 1 rules/lib/toplevel/testing#TestEnvironment - testing.analysis_test bzl:rule 1 rules/lib/toplevel/testing#analysis_test - -toolchain_type bzl:type 1 ules/lib/builtins/toolchain_type.html - +toolchain bzl:rule 1 reference/be/platforms-and-toolchains#toolchain - +toolchain.exec_compatible_with bzl:rule 1 reference/be/platforms-and-toolchains#toolchain.exec_compatible_with - +toolchain.target_settings bzl:attribute 1 reference/be/platforms-and-toolchains#toolchain.target_settings - +toolchain.target_compatible_with bzl:attribute 1 reference/be/platforms-and-toolchains#toolchain.target_compatible_with - +toolchain_type bzl:type 1 rules/lib/builtins/toolchain_type.html - diff --git a/sphinxdocs/src/sphinx_bzl/bzl.py b/sphinxdocs/src/sphinx_bzl/bzl.py index 2980ecb21e..cbd35a958b 100644 --- a/sphinxdocs/src/sphinx_bzl/bzl.py +++ b/sphinxdocs/src/sphinx_bzl/bzl.py @@ -34,7 +34,7 @@ from sphinx.util import inspect, logging from sphinx.util import nodes as sphinx_nodes from sphinx.util import typing as sphinx_typing -from typing_extensions import override, TypeAlias +from typing_extensions import TypeAlias, override _logger = logging.getLogger(__name__) _LOG_PREFIX = f"[{_logger.name}] " @@ -552,7 +552,9 @@ def before_content(self) -> None: @override def transform_content(self, content_node: addnodes.desc_content) -> None: - def first_child_with_class_name(root, class_name) -> typing.Union[None, docutils_nodes.Element]: + def first_child_with_class_name( + root, class_name + ) -> typing.Union[None, docutils_nodes.Element]: matches = root.findall( lambda node: isinstance(node, docutils_nodes.Element) and class_name in node["classes"] @@ -1437,7 +1439,9 @@ class _BzlDomain(domains.Domain): object_types = { "arg": domains.ObjType("arg", "arg", "obj"), # macro/function arg "aspect": domains.ObjType("aspect", "aspect", "obj"), - "attribute": domains.ObjType("attribute", "attribute", "obj"), # rule attribute + "attribute": domains.ObjType( + "attribute", "attribute", "attr", "obj" + ), # rule attribute "function": domains.ObjType("function", "func", "obj"), "method": domains.ObjType("method", "method", "obj"), "module-extension": domains.ObjType( @@ -1509,7 +1513,9 @@ class _BzlDomain(domains.Domain): } @override - def get_full_qualified_name(self, node: docutils_nodes.Element) -> typing.Union[str, None]: + def get_full_qualified_name( + self, node: docutils_nodes.Element + ) -> typing.Union[str, None]: bzl_file = node.get("bzl:file") symbol_name = node.get("bzl:symbol") ref_target = node.get("reftarget") @@ -1574,6 +1580,10 @@ def _find_entry_for_xref( if target.startswith("--"): target = target.strip("-") object_type = "flag" + + # Allow using parentheses, e.g. `foo()` or `foo(x=...)` + target, _, _ = target.partition("(") + # Elide the value part of --foo=bar flags # Note that the flag value could contain `=` if "=" in target: