diff --git a/.bazelrc b/.bazelrc index b484751c3c..1ca469cd75 100644 --- a/.bazelrc +++ b/.bazelrc @@ -4,8 +4,8 @@ # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it) # To update these lines, execute # `bazel run @rules_bazel_integration_test//tools:update_deleted_packages` -build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/python/private,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered -query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/python/private,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered +build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered +query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered test --test_output=errors diff --git a/CHANGELOG.md b/CHANGELOG.md index c4c99206b1..897210e8d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,10 @@ A brief description of the categories of changes: * (gazelle): Update error messages when unable to resolve a dependency to be more human-friendly. * (flags) The {obj}`--python_version` flag now also returns {obj}`config_common.FeatureFlagInfo`. +* (toolchain): The toolchain patches now expose the `patch_strip` attribute + that one should use when patching toolchains. Please set it if you are + patching python interpreter. In the next release the default will be set to + `0` which better reflects the defaults used in public `bazel` APIs. * (toolchains) When {obj}`py_runtime.interpreter_version_info` isn't specified, the {obj}`--python_version` flag will determine the value. This allows specifying the build-time Python version for the @@ -60,6 +64,10 @@ A brief description of the categories of changes: ### Added +* (bzlmod): Toolchain overrides can now be done using the new + {bzl:obj}`python.override`, {bzl:obj}`python.single_version_override` and + {bzl:obj}`python.single_version_platform_override` tag classes. + See [#2081](https://github.com/bazelbuild/rules_python/issues/2081). * (rules) Executables provide {obj}`PyExecutableInfo`, which contains executable-specific information useful for packaging an executable or or deriving a new one from the original. diff --git a/MODULE.bazel b/MODULE.bazel index 9ac3e7a04c..100eae8569 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -81,7 +81,7 @@ dev_python = use_extension( "python", dev_dependency = True, ) -dev_python.rules_python_private_testing( +dev_python.override( register_all_versions = True, ) diff --git a/docs/toolchains.md b/docs/toolchains.md index fac1bfc6b0..c31585079e 100644 --- a/docs/toolchains.md +++ b/docs/toolchains.md @@ -164,6 +164,17 @@ Remember to call `use_repo()` to make repos visible to your module: Python toolchains can be utilized in other bazel rules, such as `genrule()`, by adding the `toolchains=["@rules_python//python:current_py_toolchain"]` attribute. You can obtain the path to the Python interpreter using the `$(PYTHON2)` and `$(PYTHON3)` ["Make" Variables](https://bazel.build/reference/be/make-variables). See the {gh-path}`test_current_py_toolchain ` target for an example. +### Overriding toolchain defaults and adding more toolchains + +One can perform various overrides for the registered toolchains from the root module. For example, the following usecases would be supported using the existing attributes: + +* Limiting the available toolchains for the entire `bzlmod` transitive graph + via {attr}`python.override.available_python_versions`. +* Setting particular `X.Y.Z` python versions when modules request `X.Y` version + via {attr}`python.override.minor_mapping`. +* Adding custom {attr}`python.single_version_platform_override.coverage_tool`. +* Adding new python versions via {bzl:obj}`python.single_version_override` or + {bzl:obj}`python.single_version_platform_override`. ## Workspace configuration @@ -240,5 +251,5 @@ automatically registers a higher-priority toolchain; it won't be used unless there is a toolchain misconfiguration somewhere. To aid migration off the Bazel-builtin toolchain, rules_python provides -{obj}`@rules_python//python/runtime_env_toolchains:all`. This is an equivalent +{bzl:obj}`@rules_python//python/runtime_env_toolchains:all`. This is an equivalent toolchain, but is implemented using rules_python's objects. diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index b7b46b7dba..502c6fa712 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -22,7 +22,7 @@ bazel_dep(name = "protobuf", version = "24.4", repo_name = "com_google_protobuf" python = use_extension("@rules_python//python/extensions:python.bzl", "python") python.toolchain( configure_coverage_tool = True, - # Only set when you have mulitple toolchain versions. + # Only set when you have multiple toolchain versions. is_default = True, python_version = "3.9", ) @@ -37,6 +37,54 @@ python.toolchain( python_version = "3.10", ) +# One can override the actual toolchain versions that are available, which can be useful +# to when optimizing what gets downloaded and when. +python.override( + available_python_versions = [ + "3.10.9", + "3.9.19", + # The following is used by the `other_module` and we need to include it here + # as well. + "3.11.8", + ], + # Also override the `minor_mapping` so that when the modules specify a particular + # `3.X` version, we decide what gets used. + minor_mapping = { + "3.10": "3.10.9", + "3.11": "3.11.8", + "3.9": "3.9.19", + }, +) + +# Or the sources that the toolchains come from for all platforms +python.single_version_override( + patch_strip = 1, + # The user can specify patches to be applied to all interpreters. + patches = [], + python_version = "3.10.2", + sha256 = { + "aarch64-apple-darwin": "1409acd9a506e2d1d3b65c1488db4e40d8f19d09a7df099667c87a506f71c0ef", + "aarch64-unknown-linux-gnu": "8f351a8cc348bb45c0f95b8634c8345ec6e749e483384188ad865b7428342703", + "x86_64-apple-darwin": "8146ad4390710ec69b316a5649912df0247d35f4a42e2aa9615bffd87b3e235a", + "x86_64-pc-windows-msvc": "a1d9a594cd3103baa24937ad9150c1a389544b4350e859200b3e5c036ac352bd", + "x86_64-unknown-linux-gnu": "9b64eca2a94f7aff9409ad70bdaa7fbbf8148692662e764401883957943620dd", + }, + urls = ["20220227/cpython-{python_version}+20220227-{platform}-{build}.tar.gz"], +) + +# Or a single platform. This can be used in combination with the +# `single_version_override` and `single_version_platform_override` will be +# applied after `single_version_override`. Any values present in this override +# will overwrite the values set by the `single_version_override` +python.single_version_platform_override( + patch_strip = 1, + patches = [], + platform = "aarch64-apple-darwin", + python_version = "3.10.2", + sha256 = "1409acd9a506e2d1d3b65c1488db4e40d8f19d09a7df099667c87a506f71c0ef", + urls = ["20220227/cpython-{python_version}+20220227-{platform}-{build}.tar.gz"], +) + # You only need to load this repositories if you are using multiple Python versions. # See the tests folder for various examples on using multiple Python versions. # The names "python_3_9" and "python_3_10" are autmatically created by the repo diff --git a/examples/bzlmod/MODULE.bazel.lock b/examples/bzlmod/MODULE.bazel.lock index 0cfe49d5d8..d3845e8b98 100644 --- a/examples/bzlmod/MODULE.bazel.lock +++ b/examples/bzlmod/MODULE.bazel.lock @@ -1231,7 +1231,7 @@ }, "@@rules_python~//python/extensions:pip.bzl%pip": { "general": { - "bzlTransitiveDigest": "7vRndkQ5a5Q2gcPIP8Jd/AkNRuB4n7SofpNFmFvodG8=", + "bzlTransitiveDigest": "JSi8I0KuNjiOs/YNqHpBvZsZ4IQoMoslQoz7iyklQlE=", "usagesDigest": "MChlcSw99EuW3K7OOoMcXQIdcJnEh6YmfyjJm+9mxIg=", "recordedFileInputs": { "@@other_module~//requirements_lock_3_11.txt": "a7d0061366569043d5efcf80e34a32c732679367cb3c831c4cdc606adc36d314", @@ -6140,7 +6140,7 @@ }, "@@rules_python~//python/private/pypi:pip.bzl%pip_internal": { "general": { - "bzlTransitiveDigest": "DQe4hZM+myEcJ/pVW54jl5vWJOw+oZNBZfE0WOX/S9g=", + "bzlTransitiveDigest": "lziW178DzsD+yErlQGWP0cHgVs1WS7PN029k70hxdl0=", "usagesDigest": "Y8ihY+R57BAFhalrVLVdJFrpwlbsiKz9JPJ99ljF7HA=", "recordedFileInputs": { "@@rules_python~//tools/publish/requirements.txt": "031e35d03dde03ae6305fe4b3d1f58ad7bdad857379752deede0f93649991b8a", diff --git a/python/extensions/python.bzl b/python/extensions/python.bzl index 4148d90877..ce98dc3fec 100644 --- a/python/extensions/python.bzl +++ b/python/extensions/python.bzl @@ -12,7 +12,34 @@ # See the License for the specific language governing permissions and # limitations under the License. -"Python toolchain module extensions for use with bzlmod" +"""Python toolchain module extensions for use with bzlmod. + +## Basic usage + +The simplest way to configure the toolchain with `rules_python` is as follows. + +```starlark +python = use_extension("@rules_python//python/extensions:python.bzl", "python") +python.toolchain( + is_default = True, + python_version = "3.11", +) +use_repo(python, "python_3_11") +``` + +For more in-depth documentation see the {rule}`python.toolchain`. + +## Overrides + +Overrides can be done at 3 different levels: +* Overrides affecting all python toolchain versions on all platforms - {obj}`python.override`. +* Overrides affecting a single toolchain versions on all platforms - {obj}`python.single_version_override`. +* Overrides affecting a single toolchain versions on a single platforms - {obj}`python.single_version_platform_override`. + +:::{seealso} +The main documentation page on registering [toolchains](/toolchains). +::: +""" load("//python/private:python.bzl", _python = "python") diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index a35e2f7c2e..a9ea0cb537 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -132,11 +132,12 @@ bzl_library( srcs = ["python.bzl"], deps = [ ":full_version_bzl", + ":python_repositories_bzl", ":pythons_hub_bzl", ":repo_utils_bzl", + ":semver_bzl", ":toolchains_repo_bzl", ":util_bzl", - "//python:repositories_bzl", "@bazel_features//:features", ], ) @@ -162,7 +163,6 @@ bzl_library( name = "pythons_hub_bzl", srcs = ["pythons_hub.bzl"], deps = [ - ":full_version_bzl", ":py_toolchain_suite_bzl", ], ) diff --git a/python/private/common/py_runtime_rule.bzl b/python/private/common/py_runtime_rule.bzl index dd40f76a00..b339425099 100644 --- a/python/private/common/py_runtime_rule.bzl +++ b/python/private/common/py_runtime_rule.bzl @@ -202,8 +202,8 @@ See @bazel_tools//tools/python:python_bootstrap_template.txt for more variables. "coverage_tool": attr.label( allow_files = False, doc = """ -This is a target to use for collecting code coverage information from `py_binary` -and `py_test` targets. +This is a target to use for collecting code coverage information from +{rule}`py_binary` and {rule}`py_test` targets. If set, the target must either produce a single file or be an executable target. The path to the single file, or the executable if the target is executable, @@ -212,7 +212,7 @@ runfiles will be added to the runfiles when coverage is enabled. The entry point for the tool must be loadable by a Python interpreter (e.g. a `.py` or `.pyc` file). It must accept the command line arguments -of coverage.py (https://coverage.readthedocs.io), at least including +of [`coverage.py`](https://coverage.readthedocs.io), at least including the `run` and `lcov` subcommands. """, ), @@ -311,7 +311,7 @@ The template to use when two stage bootstrapping is enabled default = DEFAULT_STUB_SHEBANG, doc = """ "Shebang" expression prepended to the bootstrapping Python stub script -used when executing `py_binary` targets. +used when executing {rule}`py_binary` targets. See https://github.com/bazelbuild/bazel/issues/8685 for motivation. diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 1db50af7c7..c33514db61 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -59,7 +59,6 @@ bzl_library( srcs = ["extension.bzl"], deps = [ ":attrs_bzl", - "//python/private:semver_bzl", ":hub_repository_bzl", ":parse_requirements_bzl", ":evaluate_markers_bzl", @@ -71,6 +70,7 @@ bzl_library( "//python/private:full_version_bzl", "//python/private:normalize_name_bzl", "//python/private:version_label_bzl", + "//python/private:semver_bzl", "@bazel_features//:features", ] + [ "@pythons_hub//:interpreters_bzl", diff --git a/python/private/python.bzl b/python/private/python.bzl index 9a9a240cb3..90f630586b 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -12,14 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -"Python toolchain module extensions for use with bzlmod" +"Python toolchain module extensions for use with bzlmod." load("@bazel_features//:features.bzl", "bazel_features") -load("//python:repositories.bzl", "python_register_toolchains") -load("//python:versions.bzl", "MINOR_MAPPING", "TOOL_VERSIONS") +load("//python:versions.bzl", "DEFAULT_RELEASE_BASE_URL", "PLATFORMS", "TOOL_VERSIONS") +load(":auth.bzl", "AUTH_ATTRS") load(":full_version.bzl", "full_version") +load(":python_repositories.bzl", "python_register_toolchains") load(":pythons_hub.bzl", "hub_repo") load(":repo_utils.bzl", "repo_utils") +load(":semver.bzl", "semver") load(":text_util.bzl", "render") load(":toolchains_repo.bzl", "multi_toolchain_aliases") load(":util.bzl", "IS_BAZEL_6_4_OR_HIGHER") @@ -29,11 +31,12 @@ load(":util.bzl", "IS_BAZEL_6_4_OR_HIGHER") _MAX_NUM_TOOLCHAINS = 9999 _TOOLCHAIN_INDEX_PAD_LENGTH = len(str(_MAX_NUM_TOOLCHAINS)) -def parse_modules(module_ctx): +def parse_modules(*, module_ctx, _fail = fail): """Parse the modules and return a struct for registrations. Args: module_ctx: {type}`module_ctx` module context. + _fail: {type}`function` the failure function, mainly for testing. Returns: A struct with the following attributes: @@ -61,7 +64,7 @@ def parse_modules(module_ctx): # This is a toolchain_info struct. default_toolchain = None - # Map of version string to the toolchain_info struct + # Map of string Major.Minor or Major.Minor.Patch to the toolchain_info struct global_toolchain_versions = {} ignore_root_user_error = None @@ -73,10 +76,16 @@ def parse_modules(module_ctx): if not module_ctx.modules[0].tags.toolchain: ignore_root_user_error = False + config = _get_toolchain_config(modules = module_ctx.modules, _fail = _fail) + + seen_versions = {} for mod in module_ctx.modules: module_toolchain_versions = [] - - toolchain_attr_structs = _create_toolchain_attr_structs(mod) + toolchain_attr_structs = _create_toolchain_attr_structs( + mod = mod, + seen_versions = seen_versions, + config = config, + ) for toolchain_attr in toolchain_attr_structs: toolchain_version = toolchain_attr.python_version @@ -166,6 +175,8 @@ def parse_modules(module_ctx): elif toolchain_info: toolchains.append(toolchain_info) + config.default.setdefault("ignore_root_user_error", ignore_root_user_error) + # A default toolchain is required so that the non-version-specific rules # are able to match a toolchain. if default_toolchain == None: @@ -185,11 +196,9 @@ def parse_modules(module_ctx): fail("more than {} python versions are not supported".format(_MAX_NUM_TOOLCHAINS)) return struct( + config = config, debug_info = debug_info, default_python_version = toolchains[-1].python_version, - defaults = { - "ignore_root_user_error": ignore_root_user_error, - }, toolchains = [ struct( python_version = t.python_version, @@ -201,16 +210,24 @@ def parse_modules(module_ctx): ) def _python_impl(module_ctx): - py = parse_modules(module_ctx) + py = parse_modules(module_ctx = module_ctx) for toolchain_info in py.toolchains: - python_register_toolchains( - name = toolchain_info.name, - python_version = toolchain_info.python_version, - register_coverage_tool = toolchain_info.register_coverage_tool, - minor_mapping = MINOR_MAPPING, - **py.defaults + # Ensure that we pass the full version here. + full_python_version = full_version( + version = toolchain_info.python_version, + minor_mapping = py.config.minor_mapping, ) + kwargs = { + "python_version": full_python_version, + "register_coverage_tool": toolchain_info.register_coverage_tool, + } + + # Allow overrides per python version + kwargs.update(py.config.kwargs.get(toolchain_info.python_version, {})) + kwargs.update(py.config.kwargs.get(full_python_version, {})) + kwargs.update(py.config.default) + python_register_toolchains(name = toolchain_info.name, **kwargs) # Create the pythons_hub repo for the interpreter meta data and the # the various toolchains. @@ -223,7 +240,7 @@ def _python_impl(module_ctx): for index, toolchain in enumerate(py.toolchains) ], toolchain_python_versions = [ - full_version(version = t.python_version, minor_mapping = MINOR_MAPPING) + full_version(version = t.python_version, minor_mapping = py.config.minor_mapping) for t in py.toolchains ], # The last toolchain is the default; it can't have version constraints @@ -264,6 +281,9 @@ def _fail_duplicate_module_toolchain_version(version, module): )) def _warn_duplicate_global_toolchain_version(version, first, second_toolchain_name, second_module_name, logger): + if not logger: + return + logger.info(lambda: ( "Ignoring toolchain '{second_toolchain}' from module '{second_module}': " + "Toolchain '{first_toolchain}' from module '{first_module}' " + @@ -284,25 +304,234 @@ def _fail_multiple_default_toolchains(first, second): second = second, )) -def _create_toolchain_attr_structs(mod): +def _validate_version(*, version, _fail = fail): + parsed = semver(version) + if not parsed.patch or parsed.build: + _fail("The 'python_version' attribute needs to specify an 'X.Y.Z' semver-compatible version, got: '{}'".format(version)) + return False + + return True + +def _process_single_version_overrides(*, tag, _fail = fail, default): + if not _validate_version(version = tag.python_version, _fail = _fail): + return + + available_versions = default["tool_versions"] + kwargs = default.setdefault("kwargs", {}) + + if tag.sha256 or tag.urls: + if not (tag.sha256 and tag.urls): + _fail("Both `sha256` and `urls` overrides need to be provided together") + return + + for platform in tag.sha256 or []: + if platform not in PLATFORMS: + _fail("The platform must be one of {allowed} but got '{got}'".format( + allowed = sorted(PLATFORMS), + got = platform, + )) + return + + sha256 = dict(tag.sha256) or available_versions[tag.python_version]["sha256"] + override = { + "sha256": sha256, + "strip_prefix": { + platform: tag.strip_prefix + for platform in sha256 + }, + "url": { + platform: list(tag.urls) + for platform in tag.sha256 + } or available_versions[tag.python_version]["url"], + } + + if tag.patches: + override["patch_strip"] = { + platform: tag.patch_strip + for platform in sha256 + } + override["patches"] = { + platform: list(tag.patches) + for platform in sha256 + } + + available_versions[tag.python_version] = {k: v for k, v in override.items() if v} + + if tag.distutils_content: + kwargs.setdefault(tag.python_version, {})["distutils_content"] = tag.distutils_content + if tag.distutils: + kwargs.setdefault(tag.python_version, {})["distutils"] = tag.distutils + +def _process_single_version_platform_overrides(*, tag, _fail = fail, default): + if not _validate_version(version = tag.python_version, _fail = _fail): + return + + available_versions = default["tool_versions"] + + if tag.python_version not in available_versions: + if not tag.urls or not tag.sha256 or not tag.strip_prefix: + _fail("When introducing a new python_version '{}', 'sha256', 'strip_prefix' and 'urls' must be specified".format(tag.python_version)) + return + available_versions[tag.python_version] = {} + + if tag.coverage_tool: + available_versions[tag.python_version].setdefault("coverage_tool", {})[tag.platform] = tag.coverage_tool + if tag.patch_strip: + available_versions[tag.python_version].setdefault("patch_strip", {})[tag.platform] = tag.patch_strip + if tag.patches: + available_versions[tag.python_version].setdefault("patches", {})[tag.platform] = list(tag.patches) + if tag.sha256: + available_versions[tag.python_version].setdefault("sha256", {})[tag.platform] = tag.sha256 + if tag.strip_prefix: + available_versions[tag.python_version].setdefault("strip_prefix", {})[tag.platform] = tag.strip_prefix + if tag.urls: + available_versions[tag.python_version].setdefault("url", {})[tag.platform] = tag.urls + +def _process_global_overrides(*, tag, default, _fail = fail): + if tag.available_python_versions: + available_versions = default["tool_versions"] + all_versions = dict(available_versions) + available_versions.clear() + for v in tag.available_python_versions: + if v not in all_versions: + _fail("unknown version '{}', known versions are: {}".format( + v, + sorted(all_versions), + )) + return + + available_versions[v] = all_versions[v] + + if tag.minor_mapping: + for minor_version, full_version in tag.minor_mapping.items(): + parsed = semver(minor_version) + if parsed.patch or parsed.build: + fail("Expected the key to be of `X.Y` format but got `{}`".format(minor_version)) + parsed = semver(full_version) + if not parsed.patch: + fail("Expected the value to at least be of `X.Y.Z` format but got `{}`".format(minor_version)) + + default["minor_mapping"] = tag.minor_mapping + + forwarded_attrs = sorted(AUTH_ATTRS) + [ + "ignore_root_user_error", + "base_url", + "register_all_versions", + ] + for key in forwarded_attrs: + if getattr(tag, key, None): + default[key] = getattr(tag, key) + +def _override_defaults(*overrides, modules, _fail = fail, default): + mod = modules[0] if modules else None + if not mod or not mod.is_root: + return + + overriden_keys = [] + + for override in overrides: + for tag in getattr(mod.tags, override.name): + key = override.key(tag) + if key not in overriden_keys: + overriden_keys.append(key) + elif key: + _fail("Only a single 'python.{}' can be present for '{}'".format(override.name, key)) + return + else: + _fail("Only a single 'python.{}' can be present".format(override.name)) + return + + override.fn(tag = tag, _fail = _fail, default = default) + +def _get_toolchain_config(*, modules, _fail = fail): + # Items that can be overridden + available_versions = { + version: { + # Use a dicts straight away so that we could do URL overrides for a + # single version. + "sha256": dict(item["sha256"]), + "strip_prefix": { + platform: item["strip_prefix"] + for platform in item["sha256"] + }, + "url": { + platform: [item["url"]] + for platform in item["sha256"] + }, + } + for version, item in TOOL_VERSIONS.items() + } + default = { + "base_url": DEFAULT_RELEASE_BASE_URL, + "tool_versions": available_versions, + } + + _override_defaults( + # First override by single version, because the sha256 will replace + # anything that has been there before. + struct( + name = "single_version_override", + key = lambda t: t.python_version, + fn = _process_single_version_overrides, + ), + # Then override particular platform entries if they need to be overridden. + struct( + name = "single_version_platform_override", + key = lambda t: (t.python_version, t.platform), + fn = _process_single_version_platform_overrides, + ), + # Then finally add global args and remove the unnecessary toolchains. + # This ensures that we can do further validations when removing. + struct( + name = "override", + key = lambda t: None, + fn = _process_global_overrides, + ), + modules = modules, + default = default, + _fail = _fail, + ) + + minor_mapping = default.pop("minor_mapping", {}) + register_all_versions = default.pop("register_all_versions", False) + kwargs = default.pop("kwargs", {}) + + if not minor_mapping: + versions = {} + for version_string in available_versions: + v = semver(version_string) + versions.setdefault("{}.{}".format(v.major, v.minor), []).append((int(v.patch), version_string)) + + minor_mapping = { + major_minor: max(subset)[1] + for major_minor, subset in versions.items() + } + + return struct( + kwargs = kwargs, + minor_mapping = minor_mapping, + default = default, + register_all_versions = register_all_versions, + ) + +def _create_toolchain_attr_structs(*, mod, config, seen_versions): arg_structs = [] - seen_versions = {} + for tag in mod.tags.toolchain: - arg_structs.append(_create_toolchain_attrs_struct(tag = tag, toolchain_tag_count = len(mod.tags.toolchain))) + arg_structs.append(_create_toolchain_attrs_struct( + tag = tag, + toolchain_tag_count = len(mod.tags.toolchain), + )) + seen_versions[tag.python_version] = True - if mod.is_root: - register_all = False - for tag in mod.tags.rules_python_private_testing: - if tag.register_all_versions: - register_all = True - break - if register_all: - arg_structs.extend([ - _create_toolchain_attrs_struct(python_version = v) - for v in TOOL_VERSIONS.keys() - if v not in seen_versions - ]) + if config.register_all_versions: + arg_structs.extend([ + _create_toolchain_attrs_struct(python_version = v) + for v in config.default["tool_versions"].keys() + config.minor_mapping.keys() + if v not in seen_versions + ]) + return arg_structs def _create_toolchain_attrs_struct(*, tag = None, python_version = None, toolchain_tag_count = None): @@ -329,79 +558,299 @@ def _get_bazel_version_specific_kwargs(): return kwargs -python = module_extension( - doc = """Bzlmod extension that is used to register Python toolchains. -""", - implementation = _python_impl, - tag_classes = { - "rules_python_private_testing": tag_class( - attrs = { - "register_all_versions": attr.bool(default = False), - }, - ), - "toolchain": tag_class( - doc = """Tag class used to register Python toolchains. +_toolchain = tag_class( + doc = """Tag class used to register Python toolchains. Use this tag class to register one or more Python toolchains. This class is also potentially called by sub modules. The following covers different business rules and use cases. -Toolchains in the Root Module +:::{topic} Toolchains in the Root Module This class registers all toolchains in the root module. +::: -Toolchains in Sub Modules +:::{topic} Toolchains in Sub Modules It will create a toolchain that is in a sub module, if the toolchain of the same name does not exist in the root module. The extension stops name clashing between toolchains in the root module and toolchains in sub modules. You cannot configure more than one toolchain as the default toolchain. +::: -Toolchain set as the default version +:::{topic} Toolchain set as the default version This extension will not create a toolchain that exists in a sub module, if the sub module toolchain is marked as the default version. If you have more than one toolchain in your root module, you need to set one of the toolchains as the default version. If there is only one toolchain it is set as the default toolchain. +::: -Toolchain repository name +:::{topic} Toolchain repository name A toolchain's repository name uses the format `python_{major}_{minor}`, e.g. `python_3_10`. The `major` and `minor` components are `major` and `minor` are the Python version from the `python_version` attribute. + +If a toolchain is registered in `X.Y.Z`, then similarly the toolchain name will +be `python_{major}_{minor}_{patch}`, e.g. `python_3_10_19`. +::: + +:::{topic} Toolchain detection +The definition of the first toolchain wins, which means that the root module +can override settings for any python toolchain available. This relies on the +documented module traversal from the {obj}`module_ctx.modules`. +::: + +:::{tip} +In order to use a different name than the above, you can use the following `MODULE.bazel` +syntax: +```starlark +python = use_extension("@rules_python//python/extensions:python.bzl", "python") +python.toolchain( + is_default = True, + python_version = "3.11", +) + +use_repo(python, my_python_name = "python_3_11") +``` + +Then the python interpreter will be available as `my_python_name`. +::: """, - attrs = { - "configure_coverage_tool": attr.bool( - mandatory = False, - doc = "Whether or not to configure the default coverage tool for the toolchains.", - ), - "ignore_root_user_error": attr.bool( - default = False, - doc = """\ -If False, the Python runtime installation will be made read only. This improves + attrs = { + "configure_coverage_tool": attr.bool( + mandatory = False, + doc = "Whether or not to configure the default coverage tool provided by `rules_python` for the compatible toolchains.", + ), + "ignore_root_user_error": attr.bool( + default = False, + doc = """\ +If `False`, the Python runtime installation will be made read only. This improves the ability for Bazel to cache it, but prevents the interpreter from creating -pyc files for the standard library dynamically at runtime as they are loaded. +`.pyc` files for the standard library dynamically at runtime as they are loaded. -If True, the Python runtime installation is read-write. This allows the -interpreter to create pyc files for the standard library, but, because they are +If `True`, the Python runtime installation is read-write. This allows the +interpreter to create `.pyc` files for the standard library, but, because they are created as needed, it adversely affects Bazel's ability to cache the runtime and can result in spurious build failures. """, - mandatory = False, - ), - "is_default": attr.bool( - mandatory = False, - doc = "Whether the toolchain is the default version", - ), - "python_version": attr.string( - mandatory = True, - doc = "The Python version, in `major.minor` format, e.g " + - "'3.12', to create a toolchain for. Patch level " + - "granularity (e.g. '3.12.1') is not supported.", - ), - }, + mandatory = False, + ), + "is_default": attr.bool( + mandatory = False, + doc = "Whether the toolchain is the default version", + ), + "python_version": attr.string( + mandatory = True, + doc = """\ +The Python version, in `major.minor` or `major.minor.patch` format, e.g +`3.12` (or `3.12.3`), to create a toolchain for. +""", ), }, +) + +_override = tag_class( + doc = """Tag class used to override defaults and behaviour of the module extension. + +:::{versionadded} 0.36.0 +::: +""", + attrs = dict( + { + "available_python_versions": attr.string_list( + mandatory = False, + doc = """\ +The list of available python tool versions to use. Must be in `X.Y.Z` format. +If the unknown version given the processing of the extension will fail - all of +the versions in the list have to be defined with +{obj}`python.single_version_override` or +{obj}`python.single_version_platform_override` before they are used in this +list. + +This attribute is usually used in order to ensure that no unexpected transitive +dependencies are introduced. +""", + ), + "base_url": attr.string( + mandatory = False, + doc = "The base URL to be used when downloading toolchains.", + default = DEFAULT_RELEASE_BASE_URL, + ), + "ignore_root_user_error": attr.bool( + default = False, + doc = """\ +If `False`, the Python runtime installation will be made read only. This improves +the ability for Bazel to cache it, but prevents the interpreter from creating +`.pyc` files for the standard library dynamically at runtime as they are loaded. + +If `True`, the Python runtime installation is read-write. This allows the +interpreter to create `.pyc` files for the standard library, but, because they are +created as needed, it adversely affects Bazel's ability to cache the runtime and +can result in spurious build failures. +""", + mandatory = False, + ), + "minor_mapping": attr.string_dict( + mandatory = False, + doc = """\ +The mapping between `X.Y` to `X.Y.Z` versions to be used when setting up +toolchains. It defaults to the interpreter with the highest available patch +version for each minor version. For example if one registers `3.10.3`, `3.10.4` +and `3.11.4` then the default for the `minor_mapping` dict will be: +```starlark +{ + "3.10": "3.10.4", + "3.11": "3.11.4", +} +``` +""", + default = {}, + ), + "register_all_versions": attr.bool(default = False, doc = "Add all versions"), + }, + **AUTH_ATTRS + ), +) + +_single_version_override = tag_class( + doc = """Override single python version URLs and patches for all platforms. + +:::{note} +This will replace any existing configuration for the given python version. +::: + +:::{tip} +If you would like to modify the configuration for a specific `(version, +platform)`, please use the {obj}`single_version_platform_override` tag +class. +::: + +:::{versionadded} 0.36.0 +::: +""", + attrs = { + # NOTE @aignas 2024-09-01: all of the attributes except for `version` + # can be part of the `python.toolchain` call. That would make it more + # ergonomic to define new toolchains and to override values for old + # toolchains. The same semantics of the `first one wins` would apply, + # so technically there is no need for any overrides? + # + # Although these attributes would override the code that is used by the + # code in non-root modules, so technically this could be thought as + # being overridden. + # + # rules_go has a single download call: + # https://github.com/bazelbuild/rules_go/blob/master/go/private/extensions.bzl#L38 + # + # However, we need to understand how to accommodate the fact that + # {attr}`single_version_override.version` only allows patch versions. + "distutils": attr.label( + allow_single_file = True, + doc = "A distutils.cfg file to be included in the Python installation. " + + "Either {attr}`distutils` or {attr}`distutils_content` can be specified, but not both.", + mandatory = False, + ), + "distutils_content": attr.string( + doc = "A distutils.cfg file content to be included in the Python installation. " + + "Either {attr}`distutils` or {attr}`distutils_content` can be specified, but not both.", + mandatory = False, + ), + "patch_strip": attr.int( + mandatory = False, + doc = "Same as the --strip argument of Unix patch.", + default = 0, + ), + "patches": attr.label_list( + mandatory = False, + doc = "A list of labels pointing to patch files to apply for the interpreter repository. They are applied in the list order and are applied before any platform-specific patches are applied.", + ), + "python_version": attr.string( + mandatory = True, + doc = "The python version to override URLs for. Must be in `X.Y.Z` format.", + ), + "sha256": attr.string_dict( + mandatory = False, + doc = "The python platform to sha256 dict. See {attr}`python.single_version_platform_override.platform` for allowed key values.", + ), + "strip_prefix": attr.string( + mandatory = False, + doc = "The 'strip_prefix' for the archive, defaults to 'python'.", + default = "python", + ), + "urls": attr.string_list( + mandatory = False, + doc = "The URL template to fetch releases for this Python version. See {attr}`python.single_version_platform_override.urls` for documentation.", + ), + }, +) + +_single_version_platform_override = tag_class( + doc = """Override single python version for a single existing platform. + +If the `(version, platform)` is new, we will add it to the existing versions and will +use the same `url` template. + +:::{tip} +If you would like to add or remove platforms to a single python version toolchain +configuration, please use {obj}`single_version_override`. +::: + +:::{versionadded} 0.36.0 +::: +""", + attrs = { + "coverage_tool": attr.label( + doc = """\ +The coverage tool to be used for a particular Python interpreter. This can override +`rules_python` defaults. +""", + ), + "patch_strip": attr.int( + mandatory = False, + doc = "Same as the --strip argument of Unix patch.", + default = 0, + ), + "patches": attr.label_list( + mandatory = False, + doc = "A list of labels pointing to patch files to apply for the interpreter repository. They are applied in the list order and are applied after the common patches are applied.", + ), + "platform": attr.string( + mandatory = True, + values = PLATFORMS.keys(), + doc = "The platform to override the values for, must be one of:\n{}.".format("\n".join(sorted(["* `{}`".format(p) for p in PLATFORMS]))), + ), + "python_version": attr.string( + mandatory = True, + doc = "The python version to override URLs for. Must be in `X.Y.Z` format.", + ), + "sha256": attr.string( + mandatory = False, + doc = "The sha256 for the archive", + ), + "strip_prefix": attr.string( + mandatory = False, + doc = "The 'strip_prefix' for the archive, defaults to 'python'.", + default = "python", + ), + "urls": attr.string_list( + mandatory = False, + doc = "The URL template to fetch releases for this Python version. If the URL template results in a relative fragment, default base URL is going to be used. Occurrences of `{python_version}`, `{platform}` and `{build}` will be interpolated based on the contents in the override and the known {attr}`platform` values.", + ), + }, +) + +python = module_extension( + doc = """Bzlmod extension that is used to register Python toolchains. +""", + implementation = _python_impl, + tag_classes = { + "override": _override, + "single_version_override": _single_version_override, + "single_version_platform_override": _single_version_platform_override, + "toolchain": _toolchain, + }, **_get_bazel_version_specific_kwargs() ) diff --git a/python/private/python_repositories.bzl b/python/private/python_repositories.bzl index c4988ee691..51b6327683 100644 --- a/python/private/python_repositories.bzl +++ b/python/private/python_repositories.bzl @@ -48,8 +48,8 @@ def http_archive(**kwargs): def py_repositories(): """Runtime dependencies that users must install. - This function should be loaded and called in the user's WORKSPACE. - With bzlmod enabled, this function is not needed since MODULE.bazel handles transitive deps. + This function should be loaded and called in the user's `WORKSPACE`. + With `bzlmod` enabled, this function is not needed since `MODULE.bazel` handles transitive deps. """ maybe( internal_config_repo, @@ -179,7 +179,6 @@ def _python_repository_impl(rctx): patches = rctx.attr.patches if patches: for patch in patches: - # Should take the strip as an attr, but this is fine for the moment rctx.patch(patch, strip = rctx.attr.patch_strip) # Write distutils.cfg to the Python installation. @@ -475,27 +474,11 @@ python_repository = repository_rule( doc = "Override mapping of hostnames to authorization patterns; mirrors the eponymous attribute from http_archive", ), "coverage_tool": attr.string( - # Mirrors the definition at - # https://github.com/bazelbuild/bazel/blob/master/src/main/starlark/builtins_bzl/common/python/py_runtime_rule.bzl doc = """ -This is a target to use for collecting code coverage information from `py_binary` -and `py_test` targets. +This is a target to use for collecting code coverage information from {rule}`py_binary` +and {rule}`py_test` targets. -If set, the target must either produce a single file or be an executable target. -The path to the single file, or the executable if the target is executable, -determines the entry point for the python coverage tool. The target and its -runfiles will be added to the runfiles when coverage is enabled. - -The entry point for the tool must be loadable by a Python interpreter (e.g. a -`.py` or `.pyc` file). It must accept the command line arguments -of coverage.py (https://coverage.readthedocs.io), at least including -the `run` and `lcov` subcommands. - -The target is accepted as a string by the python_repository and evaluated within -the context of the toolchain repository. - -For more information see the official bazel docs -(https://bazel.build/reference/be/python#py_runtime.coverage_tool). +For more information see {attr}`py_runtime.coverage_tool`. """, ), "distutils": attr.label( @@ -586,7 +569,7 @@ def python_register_toolchains( tool_versions = None, minor_mapping = None, **kwargs): - """Convenience macro for users which does typical setup. + """Convenience macro for users which does typical setup in `WORKSPACE`. - Create a repository for each built-in platform like "python_3_8_linux_amd64" - this repository is lazily fetched when Python is needed for that platform. @@ -597,6 +580,10 @@ def python_register_toolchains( Users can avoid this macro and do these steps themselves, if they want more control. + With `bzlmod` enabled, this function is not needed since `rules_python` is + handling everything. In order to override the default behaviour from the + root module one can see the docs for the {rule}`python` extension. + Args: name: {type}`str` base name for all created repos, e.g. "python_3_8". python_version: {type}`str` the Python version. diff --git a/python/private/toolchains_repo.bzl b/python/private/toolchains_repo.bzl index 528b86fc3b..4fae987c74 100644 --- a/python/private/toolchains_repo.bzl +++ b/python/private/toolchains_repo.bzl @@ -56,9 +56,6 @@ def python_toolchain_build_file_content( build_content: Text containing toolchain definitions """ - # We create a list of toolchain content from iterating over - # the enumeration of PLATFORMS. We enumerate PLATFORMS in - # order to get us an index to increment the increment. return "\n\n".join([ """\ py_toolchain_suite( diff --git a/python/versions.bzl b/python/versions.bzl index 79e388db12..c97c1cc01f 100644 --- a/python/versions.bzl +++ b/python/versions.bzl @@ -722,5 +722,6 @@ def gen_python_config_settings(name = ""): for platform in PLATFORMS.keys(): native.config_setting( name = "{name}{platform}".format(name = name, platform = platform), + flag_values = PLATFORMS[platform].flag_values, constraint_values = PLATFORMS[platform].compatible_with, ) diff --git a/tests/integration/ignore_root_user_error/bzlmod_test.py b/tests/integration/ignore_root_user_error/bzlmod_test.py index 98715b32ec..1283415987 100644 --- a/tests/integration/ignore_root_user_error/bzlmod_test.py +++ b/tests/integration/ignore_root_user_error/bzlmod_test.py @@ -28,8 +28,16 @@ def test_toolchains(self): debug_info = json.loads(debug_path.read_bytes()) expected = [ - {"ignore_root_user_error": True, "name": "python_3_11"}, - {"ignore_root_user_error": True, "name": "python_3_10"}, + { + "ignore_root_user_error": True, + "module": {"is_root": False, "name": "submodule"}, + "name": "python_3_10", + }, + { + "ignore_root_user_error": True, + "module": {"is_root": True, "name": "ignore_root_user_error"}, + "name": "python_3_11", + }, ] self.assertCountEqual(debug_info["toolchains_registered"], expected) diff --git a/tests/python/python_tests.bzl b/tests/python/python_tests.bzl index acbd6676dc..de3c9a6762 100644 --- a/tests/python/python_tests.bzl +++ b/tests/python/python_tests.bzl @@ -15,13 +15,11 @@ "" load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/private:python.bzl", _parse_modules = "parse_modules") # buildifier: disable=bzl-visibility +load("//python:versions.bzl", "MINOR_MAPPING") +load("//python/private:python.bzl", "parse_modules") # buildifier: disable=bzl-visibility _tests = [] -def parse_modules(*, mctx, **kwargs): - return _parse_modules(module_ctx = mctx, **kwargs) - def _mock_mctx(*modules, environ = {}): return struct( os = struct(environ = environ), @@ -41,12 +39,14 @@ def _mock_mctx(*modules, environ = {}): ], ) -def _mod(*, name, toolchain = [], rules_python_private_testing = [], is_root = True): +def _mod(*, name, toolchain = [], override = [], single_version_override = [], single_version_platform_override = [], is_root = True): return struct( name = name, tags = struct( toolchain = toolchain, - rules_python_private_testing = rules_python_private_testing, + override = override, + single_version_override = single_version_override, + single_version_platform_override = single_version_platform_override, ), is_root = is_root, ) @@ -58,17 +58,88 @@ def _toolchain(python_version, *, is_default = False, **kwargs): **kwargs ) +def _override( + auth_patterns = {}, + available_python_versions = [], + base_url = "", + ignore_root_user_error = False, + minor_mapping = {}, + netrc = "", + register_all_versions = False): + return struct( + auth_patterns = auth_patterns, + available_python_versions = available_python_versions, + base_url = base_url, + ignore_root_user_error = ignore_root_user_error, + minor_mapping = minor_mapping, + netrc = netrc, + register_all_versions = register_all_versions, + ) + +def _single_version_override( + python_version = "", + sha256 = {}, + urls = [], + patch_strip = 0, + patches = [], + strip_prefix = "python", + distutils_content = "", + distutils = None): + if not python_version: + fail("missing mandatory args: python_version ({})".format(python_version)) + + return struct( + python_version = python_version, + sha256 = sha256, + urls = urls, + patch_strip = patch_strip, + patches = patches, + strip_prefix = strip_prefix, + distutils_content = distutils_content, + distutils = distutils, + ) + +def _single_version_platform_override( + coverage_tool = None, + patch_strip = 0, + patches = [], + platform = "", + python_version = "", + sha256 = "", + strip_prefix = "python", + urls = []): + if not platform or not python_version: + fail("missing mandatory args: platform ({}) and python_version ({})".format(platform, python_version)) + + return struct( + sha256 = sha256, + urls = urls, + strip_prefix = strip_prefix, + platform = platform, + coverage_tool = coverage_tool, + python_version = python_version, + patch_strip = patch_strip, + patches = patches, + ) + def _test_default(env): py = parse_modules( - mctx = _mock_mctx( + module_ctx = _mock_mctx( _mod(name = "rules_python", toolchain = [_toolchain("3.11")]), ), ) - env.expect.that_collection(py.defaults.keys()).contains_exactly([ + # The value there should be consistent in bzlmod with the automatically + # calculated value Please update the MINOR_MAPPING in //python:versions.bzl + # when this part starts failing. + env.expect.that_dict(py.config.minor_mapping).contains_exactly(MINOR_MAPPING) + env.expect.that_collection(py.config.kwargs).has_size(0) + env.expect.that_collection(py.config.default.keys()).contains_exactly([ + "base_url", "ignore_root_user_error", + "tool_versions", ]) - env.expect.that_bool(py.defaults["ignore_root_user_error"]).equals(False) + env.expect.that_bool(py.config.default["ignore_root_user_error"]).equals(False) env.expect.that_str(py.default_python_version).equals("3.11") want_toolchain = struct( @@ -82,14 +153,11 @@ _tests.append(_test_default) def _test_default_some_module(env): py = parse_modules( - mctx = _mock_mctx( + module_ctx = _mock_mctx( _mod(name = "rules_python", toolchain = [_toolchain("3.11")], is_root = False), ), ) - env.expect.that_collection(py.defaults.keys()).contains_exactly([ - "ignore_root_user_error", - ]) env.expect.that_str(py.default_python_version).equals("3.11") want_toolchain = struct( @@ -103,7 +171,7 @@ _tests.append(_test_default_some_module) def _test_default_with_patch_version(env): py = parse_modules( - mctx = _mock_mctx( + module_ctx = _mock_mctx( _mod(name = "rules_python", toolchain = [_toolchain("3.11.2")]), ), ) @@ -121,7 +189,7 @@ _tests.append(_test_default_with_patch_version) def _test_default_non_rules_python(env): py = parse_modules( - mctx = _mock_mctx( + module_ctx = _mock_mctx( # NOTE @aignas 2024-09-06: the first item in the module_ctx.modules # could be a non-root module, which is the case if the root module # does not make any calls to the extension. @@ -141,7 +209,7 @@ _tests.append(_test_default_non_rules_python) def _test_default_non_rules_python_ignore_root_user_error(env): py = parse_modules( - mctx = _mock_mctx( + module_ctx = _mock_mctx( _mod( name = "my_module", toolchain = [_toolchain("3.12", ignore_root_user_error = True)], @@ -150,7 +218,7 @@ def _test_default_non_rules_python_ignore_root_user_error(env): ), ) - env.expect.that_bool(py.defaults["ignore_root_user_error"]).equals(True) + env.expect.that_bool(py.config.default["ignore_root_user_error"]).equals(True) env.expect.that_str(py.default_python_version).equals("3.12") my_module_toolchain = struct( @@ -170,9 +238,41 @@ def _test_default_non_rules_python_ignore_root_user_error(env): _tests.append(_test_default_non_rules_python_ignore_root_user_error) +def _test_default_non_rules_python_ignore_root_user_error_override(env): + py = parse_modules( + module_ctx = _mock_mctx( + _mod( + name = "my_module", + toolchain = [_toolchain("3.12")], + override = [_override(ignore_root_user_error = True)], + ), + _mod(name = "rules_python", toolchain = [_toolchain("3.11")]), + ), + ) + + env.expect.that_bool(py.config.default["ignore_root_user_error"]).equals(True) + env.expect.that_str(py.default_python_version).equals("3.12") + + my_module_toolchain = struct( + name = "python_3_12", + python_version = "3.12", + register_coverage_tool = False, + ) + rules_python_toolchain = struct( + name = "python_3_11", + python_version = "3.11", + register_coverage_tool = False, + ) + env.expect.that_collection(py.toolchains).contains_exactly([ + rules_python_toolchain, + my_module_toolchain, + ]).in_order() + +_tests.append(_test_default_non_rules_python_ignore_root_user_error_override) + def _test_default_non_rules_python_ignore_root_user_error_non_root_module(env): py = parse_modules( - mctx = _mock_mctx( + module_ctx = _mock_mctx( _mod(name = "my_module", toolchain = [_toolchain("3.13")]), _mod(name = "some_module", toolchain = [_toolchain("3.12", ignore_root_user_error = True)]), _mod(name = "rules_python", toolchain = [_toolchain("3.11")]), @@ -180,7 +280,7 @@ def _test_default_non_rules_python_ignore_root_user_error_non_root_module(env): ) env.expect.that_str(py.default_python_version).equals("3.13") - env.expect.that_bool(py.defaults["ignore_root_user_error"]).equals(False) + env.expect.that_bool(py.config.default["ignore_root_user_error"]).equals(False) my_module_toolchain = struct( name = "python_3_13", @@ -207,7 +307,7 @@ _tests.append(_test_default_non_rules_python_ignore_root_user_error_non_root_mod def _test_first_occurance_of_the_toolchain_wins(env): py = parse_modules( - mctx = _mock_mctx( + module_ctx = _mock_mctx( _mod(name = "my_module", toolchain = [_toolchain("3.12")]), _mod(name = "some_module", toolchain = [_toolchain("3.12", configure_coverage_tool = True)]), _mod(name = "rules_python", toolchain = [_toolchain("3.11")]), @@ -235,6 +335,7 @@ def _test_first_occurance_of_the_toolchain_wins(env): rules_python_toolchain, my_module_toolchain, # default toolchain is last ]).in_order() + env.expect.that_dict(py.debug_info).contains_exactly({ "toolchains_registered": [ {"ignore_root_user_error": False, "name": "python_3_12"}, @@ -244,6 +345,355 @@ def _test_first_occurance_of_the_toolchain_wins(env): _tests.append(_test_first_occurance_of_the_toolchain_wins) +def _test_auth_overrides(env): + py = parse_modules( + module_ctx = _mock_mctx( + _mod( + name = "my_module", + toolchain = [_toolchain("3.12")], + override = [ + _override( + netrc = "/my/netrc", + auth_patterns = {"foo": "bar"}, + ), + ], + ), + _mod(name = "rules_python", toolchain = [_toolchain("3.11")]), + ), + ) + + env.expect.that_dict(py.config.default).contains_at_least({ + "auth_patterns": {"foo": "bar"}, + "ignore_root_user_error": False, + "netrc": "/my/netrc", + }) + env.expect.that_str(py.default_python_version).equals("3.12") + + my_module_toolchain = struct( + name = "python_3_12", + python_version = "3.12", + register_coverage_tool = False, + ) + rules_python_toolchain = struct( + name = "python_3_11", + python_version = "3.11", + register_coverage_tool = False, + ) + env.expect.that_collection(py.toolchains).contains_exactly([ + rules_python_toolchain, + my_module_toolchain, + ]).in_order() + +_tests.append(_test_auth_overrides) + +def _test_add_new_version(env): + py = parse_modules( + module_ctx = _mock_mctx( + _mod( + name = "my_module", + toolchain = [_toolchain("3.13")], + single_version_override = [ + _single_version_override( + python_version = "3.13.0", + sha256 = { + "aarch64-unknown-linux-gnu": "deadbeef", + }, + urls = ["example.org"], + patch_strip = 0, + patches = [], + strip_prefix = "prefix", + distutils_content = "", + distutils = None, + ), + ], + single_version_platform_override = [ + _single_version_platform_override( + sha256 = "deadb00f", + urls = ["something.org", "else.org"], + strip_prefix = "python", + platform = "aarch64-unknown-linux-gnu", + coverage_tool = "specific_cov_tool", + python_version = "3.13.1", + patch_strip = 2, + patches = ["specific-patch.txt"], + ), + ], + override = [ + _override( + base_url = "", + available_python_versions = ["3.12.4", "3.13.0", "3.13.1"], + minor_mapping = { + "3.13": "3.13.0", + }, + ), + ], + ), + ), + ) + + env.expect.that_str(py.default_python_version).equals("3.13") + env.expect.that_collection(py.config.default["tool_versions"].keys()).contains_exactly([ + "3.12.4", + "3.13.0", + "3.13.1", + ]) + env.expect.that_dict(py.config.default["tool_versions"]["3.13.0"]).contains_exactly({ + "sha256": {"aarch64-unknown-linux-gnu": "deadbeef"}, + "strip_prefix": {"aarch64-unknown-linux-gnu": "prefix"}, + "url": {"aarch64-unknown-linux-gnu": ["example.org"]}, + }) + env.expect.that_dict(py.config.default["tool_versions"]["3.13.1"]).contains_exactly({ + "coverage_tool": {"aarch64-unknown-linux-gnu": "specific_cov_tool"}, + "patch_strip": {"aarch64-unknown-linux-gnu": 2}, + "patches": {"aarch64-unknown-linux-gnu": ["specific-patch.txt"]}, + "sha256": {"aarch64-unknown-linux-gnu": "deadb00f"}, + "strip_prefix": {"aarch64-unknown-linux-gnu": "python"}, + "url": {"aarch64-unknown-linux-gnu": ["something.org", "else.org"]}, + }) + env.expect.that_dict(py.config.minor_mapping).contains_exactly({ + "3.13": "3.13.0", + }) + env.expect.that_collection(py.toolchains).contains_exactly([ + struct( + name = "python_3_13", + python_version = "3.13", + register_coverage_tool = False, + ), + ]) + +_tests.append(_test_add_new_version) + +def _test_register_all_versions(env): + py = parse_modules( + module_ctx = _mock_mctx( + _mod( + name = "my_module", + toolchain = [_toolchain("3.13")], + single_version_override = [ + _single_version_override( + python_version = "3.13.0", + sha256 = { + "aarch64-unknown-linux-gnu": "deadbeef", + }, + urls = ["example.org"], + ), + ], + single_version_platform_override = [ + _single_version_platform_override( + sha256 = "deadb00f", + urls = ["something.org"], + platform = "aarch64-unknown-linux-gnu", + python_version = "3.13.1", + ), + ], + override = [ + _override( + base_url = "", + available_python_versions = ["3.12.4", "3.13.0", "3.13.1"], + register_all_versions = True, + ), + ], + ), + ), + ) + + env.expect.that_str(py.default_python_version).equals("3.13") + env.expect.that_collection(py.config.default["tool_versions"].keys()).contains_exactly([ + "3.12.4", + "3.13.0", + "3.13.1", + ]) + env.expect.that_dict(py.config.minor_mapping).contains_exactly({ + # The mapping is calculated automatically + "3.12": "3.12.4", + "3.13": "3.13.1", + }) + env.expect.that_collection(py.toolchains).contains_exactly([ + struct( + name = name, + python_version = version, + register_coverage_tool = False, + ) + for name, version in { + "python_3_12": "3.12", + "python_3_12_4": "3.12.4", + "python_3_13": "3.13", + "python_3_13_0": "3.13.0", + "python_3_13_1": "3.13.1", + }.items() + ]) + +_tests.append(_test_register_all_versions) + +def _test_add_patches(env): + py = parse_modules( + module_ctx = _mock_mctx( + _mod( + name = "my_module", + toolchain = [_toolchain("3.13")], + single_version_override = [ + _single_version_override( + python_version = "3.13.0", + sha256 = { + "aarch64-apple-darwin": "deadbeef", + "aarch64-unknown-linux-gnu": "deadbeef", + }, + urls = ["example.org"], + patch_strip = 1, + patches = ["common.txt"], + strip_prefix = "prefix", + distutils_content = "", + distutils = None, + ), + ], + single_version_platform_override = [ + _single_version_platform_override( + sha256 = "deadb00f", + urls = ["something.org", "else.org"], + strip_prefix = "python", + platform = "aarch64-unknown-linux-gnu", + coverage_tool = "specific_cov_tool", + python_version = "3.13.0", + patch_strip = 2, + patches = ["specific-patch.txt"], + ), + ], + override = [ + _override( + base_url = "", + available_python_versions = ["3.13.0"], + minor_mapping = { + "3.13": "3.13.0", + }, + ), + ], + ), + ), + ) + + env.expect.that_str(py.default_python_version).equals("3.13") + env.expect.that_dict(py.config.default["tool_versions"]).contains_exactly({ + "3.13.0": { + "coverage_tool": {"aarch64-unknown-linux-gnu": "specific_cov_tool"}, + "patch_strip": {"aarch64-apple-darwin": 1, "aarch64-unknown-linux-gnu": 2}, + "patches": { + "aarch64-apple-darwin": ["common.txt"], + "aarch64-unknown-linux-gnu": ["specific-patch.txt"], + }, + "sha256": {"aarch64-apple-darwin": "deadbeef", "aarch64-unknown-linux-gnu": "deadb00f"}, + "strip_prefix": {"aarch64-apple-darwin": "prefix", "aarch64-unknown-linux-gnu": "python"}, + "url": { + "aarch64-apple-darwin": ["example.org"], + "aarch64-unknown-linux-gnu": ["something.org", "else.org"], + }, + }, + }) + env.expect.that_dict(py.config.minor_mapping).contains_exactly({ + "3.13": "3.13.0", + }) + env.expect.that_collection(py.toolchains).contains_exactly([ + struct( + name = "python_3_13", + python_version = "3.13", + register_coverage_tool = False, + ), + ]) + +_tests.append(_test_add_patches) + +def _test_fail_two_overrides(env): + errors = [] + parse_modules( + module_ctx = _mock_mctx( + _mod( + name = "my_module", + toolchain = [_toolchain("3.13")], + override = [ + _override(base_url = "foo"), + _override(base_url = "bar"), + ], + ), + ), + _fail = errors.append, + ) + env.expect.that_collection(errors).contains_exactly([ + "Only a single 'python.override' can be present", + ]) + +_tests.append(_test_fail_two_overrides) + +def _test_single_version_override_errors(env): + for test in [ + struct( + overrides = [ + _single_version_override(python_version = "3.12.4", distutils_content = "foo"), + _single_version_override(python_version = "3.12.4", distutils_content = "foo"), + ], + want_error = "Only a single 'python.single_version_override' can be present for '3.12.4'", + ), + struct( + overrides = [ + _single_version_override(python_version = "3.12.4+3", distutils_content = "foo"), + ], + want_error = "The 'python_version' attribute needs to specify an 'X.Y.Z' semver-compatible version, got: '3.12.4+3'", + ), + ]: + errors = [] + parse_modules( + module_ctx = _mock_mctx( + _mod( + name = "my_module", + toolchain = [_toolchain("3.13")], + single_version_override = test.overrides, + ), + ), + _fail = errors.append, + ) + env.expect.that_collection(errors).contains_exactly([test.want_error]) + +_tests.append(_test_single_version_override_errors) + +def _test_single_version_platform_override_errors(env): + for test in [ + struct( + overrides = [ + _single_version_platform_override(python_version = "3.12.4", platform = "foo", coverage_tool = "foo"), + _single_version_platform_override(python_version = "3.12.4", platform = "foo", coverage_tool = "foo"), + ], + want_error = "Only a single 'python.single_version_platform_override' can be present for '(\"3.12.4\", \"foo\")'", + ), + struct( + overrides = [ + _single_version_platform_override(python_version = "3.12", platform = "foo"), + ], + want_error = "The 'python_version' attribute needs to specify an 'X.Y.Z' semver-compatible version, got: '3.12'", + ), + struct( + overrides = [ + _single_version_platform_override(python_version = "3.12.1+my_build", platform = "foo"), + ], + want_error = "The 'python_version' attribute needs to specify an 'X.Y.Z' semver-compatible version, got: '3.12.1+my_build'", + ), + ]: + errors = [] + parse_modules( + module_ctx = _mock_mctx( + _mod( + name = "my_module", + toolchain = [_toolchain("3.13")], + single_version_platform_override = test.overrides, + ), + ), + _fail = errors.append, + ) + env.expect.that_collection(errors).contains_exactly([test.want_error]) + +_tests.append(_test_single_version_platform_override_errors) + +# TODO @aignas 2024-09-03: add failure tests: +# * incorrect platform failure +# * missing python_version failure + def python_test_suite(name): """Create the test suite.