From 3ed3ff1985b1502a1d0efb2d4ad3730d0e5249cf Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 27 Aug 2024 13:29:37 -0400 Subject: [PATCH] build: add release scripts using kokoro (#236) Co-authored-by: @vishwarajanand Co-authored-by: @andrewsg --- .kokoro/build.sh | 52 ------------------- .kokoro/release-manylinux-wheel.sh | 10 ++++ .kokoro/release-osx-wheel.sh | 11 ++++ .kokoro/release-windows-wheel.bat | 44 ++++++++++++++++ .kokoro/release/linux/common.cfg | 44 ++++++++++++++++ .kokoro/release/linux/linux_wheel.cfg | 14 +++++ .kokoro/release/osx/common.cfg | 44 ++++++++++++++++ .kokoro/release/osx/osx-slcn_wheel.cfg | 14 +++++ .kokoro/release/osx/osx_wheel.cfg | 14 +++++ .kokoro/release/windows/common.cfg | 44 ++++++++++++++++ .kokoro/release/windows/windows_wheel.cfg | 14 +++++ owlbot.py | 3 +- scripts/manylinux/build.sh | 4 ++ .../manylinux/publish_python_wheel.sh | 10 ++-- scripts/osx/build_python_wheel.sh | 4 ++ scripts/osx/publish_python_wheel.sh | 31 +++++++++++ 16 files changed, 299 insertions(+), 58 deletions(-) delete mode 100755 .kokoro/build.sh create mode 100755 .kokoro/release-manylinux-wheel.sh create mode 100755 .kokoro/release-osx-wheel.sh create mode 100755 .kokoro/release-windows-wheel.bat create mode 100644 .kokoro/release/linux/common.cfg create mode 100644 .kokoro/release/linux/linux_wheel.cfg create mode 100644 .kokoro/release/osx/common.cfg create mode 100644 .kokoro/release/osx/osx-slcn_wheel.cfg create mode 100644 .kokoro/release/osx/osx_wheel.cfg create mode 100644 .kokoro/release/windows/common.cfg create mode 100644 .kokoro/release/windows/windows_wheel.cfg rename .kokoro/release.sh => scripts/manylinux/publish_python_wheel.sh (71%) create mode 100755 scripts/osx/publish_python_wheel.sh diff --git a/.kokoro/build.sh b/.kokoro/build.sh deleted file mode 100755 index 4c3e8052..00000000 --- a/.kokoro/build.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eo pipefail - -if [[ -z "${PROJECT_ROOT:-}" ]]; then - PROJECT_ROOT="github/python-crc32c" -fi - -cd "${PROJECT_ROOT}" - -# Disable buffering, so that the logs stream through. -export PYTHONUNBUFFERED=1 - -# Debug: show build environment -env | grep KOKORO - -# Setup service account credentials. -export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-account.json - -# Setup project id. -export PROJECT_ID=$(cat "${KOKORO_GFILE_DIR}/project-id.json") - -# If this is a continuous build, send the test log to the FlakyBot. -# See https://github.com/googleapis/repo-automation-bots/tree/main/packages/flakybot. -if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"continuous"* ]]; then - cleanup() { - chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot - $KOKORO_GFILE_DIR/linux_amd64/flakybot - } - trap cleanup EXIT HUP -fi - -# If NOX_SESSION is set, it only runs the specified session, -# otherwise run all the sessions. -if [[ -n "${NOX_SESSION:-}" ]]; then - python3 -m nox -s ${NOX_SESSION:-} -else - python3 -m nox -fi diff --git a/.kokoro/release-manylinux-wheel.sh b/.kokoro/release-manylinux-wheel.sh new file mode 100755 index 00000000..9a3d913d --- /dev/null +++ b/.kokoro/release-manylinux-wheel.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -eo pipefail + +cd github/python-crc32c + +export PUBLISH_WHEELS="true" + +# Build and publish ManyLinux wheels +./scripts/manylinux/build.sh diff --git a/.kokoro/release-osx-wheel.sh b/.kokoro/release-osx-wheel.sh new file mode 100755 index 00000000..a33dce92 --- /dev/null +++ b/.kokoro/release-osx-wheel.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -eo pipefail + +REPO_ROOT=$(pwd)/github/python-crc32c +cd github/python-crc32c + +export PUBLISH_WHEELS="true" + +# Build and publish OSX wheels +./scripts/osx/build.sh diff --git a/.kokoro/release-windows-wheel.bat b/.kokoro/release-windows-wheel.bat new file mode 100755 index 00000000..d3564e1d --- /dev/null +++ b/.kokoro/release-windows-wheel.bat @@ -0,0 +1,44 @@ +@rem Copyright 2019 Google LLC. All rights reserved. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. + +@echo "Starting Windows release" + +cd /d %~dp0 +cd .. + +@echo "Build Wheel" +call scripts\windows\build.bat || goto :error + +@echo "Start the releasetool reporter" +call py -3 -m pip install gcp-releasetool || goto :error +call py -3 -m releasetool publish-reporter-script > C:\temp\publisher-script || goto :error + +@echo "Ensure that we have the latest versions of Twine, Wheel, and Setuptools." +call py -3 -m pip install --upgrade twine wheel setuptools || goto :error + +@echo "Disable buffering, so that the logs stream through." +set PYTHONUNBUFFERED=1 + +@echo "## RELASE WORKFLOW SUCCESSFUL ##" +@echo "## Uploading Wheels ##" + +@echo "Move into the package, build the distribution and upload." +set /p TWINE_PASSWORD=<%KOKORO_KEYSTORE_DIR%/73713_google-cloud-pypi-token-keystore-1 +call py -3 setup.py sdist || goto :error +call py -3 -m twine upload --skip-existing --username __token__ --password "%TWINE_PASSWORD%" dist/* wheels/google_crc32c* || goto :error + +goto :EOF + +:error +exit /b 1 diff --git a/.kokoro/release/linux/common.cfg b/.kokoro/release/linux/common.cfg new file mode 100644 index 00000000..694320c7 --- /dev/null +++ b/.kokoro/release/linux/common.cfg @@ -0,0 +1,44 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "python-crc32c/.kokoro/trampoline.sh" + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-multi" +} +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-crc32c/.kokoro/release.sh" +} + +# Fetch the token needed for reporting release status to GitHub +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "yoshi-automation-github-key" + } + } +} + +# Fetch PyPI password +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "google-cloud-pypi-token-keystore-1" + } + } +} diff --git a/.kokoro/release/linux/linux_wheel.cfg b/.kokoro/release/linux/linux_wheel.cfg new file mode 100644 index 00000000..0c32ba5e --- /dev/null +++ b/.kokoro/release/linux/linux_wheel.cfg @@ -0,0 +1,14 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-crc32c/.kokoro/release-manylinux-wheel.sh" +} + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "crc32c" +} + +build_file: "python-crc32c/.kokoro/release-manylinux-wheel.sh" diff --git a/.kokoro/release/osx/common.cfg b/.kokoro/release/osx/common.cfg new file mode 100644 index 00000000..694320c7 --- /dev/null +++ b/.kokoro/release/osx/common.cfg @@ -0,0 +1,44 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "python-crc32c/.kokoro/trampoline.sh" + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-multi" +} +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-crc32c/.kokoro/release.sh" +} + +# Fetch the token needed for reporting release status to GitHub +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "yoshi-automation-github-key" + } + } +} + +# Fetch PyPI password +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "google-cloud-pypi-token-keystore-1" + } + } +} diff --git a/.kokoro/release/osx/osx-slcn_wheel.cfg b/.kokoro/release/osx/osx-slcn_wheel.cfg new file mode 100644 index 00000000..3a40d7a8 --- /dev/null +++ b/.kokoro/release/osx/osx-slcn_wheel.cfg @@ -0,0 +1,14 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-crc32c/.kokoro/release-osx-wheel.sh" +} + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "crc32c" +} + +build_file: "python-crc32c/.kokoro/release-osx-wheel.sh" diff --git a/.kokoro/release/osx/osx_wheel.cfg b/.kokoro/release/osx/osx_wheel.cfg new file mode 100644 index 00000000..3a40d7a8 --- /dev/null +++ b/.kokoro/release/osx/osx_wheel.cfg @@ -0,0 +1,14 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-crc32c/.kokoro/release-osx-wheel.sh" +} + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "crc32c" +} + +build_file: "python-crc32c/.kokoro/release-osx-wheel.sh" diff --git a/.kokoro/release/windows/common.cfg b/.kokoro/release/windows/common.cfg new file mode 100644 index 00000000..694320c7 --- /dev/null +++ b/.kokoro/release/windows/common.cfg @@ -0,0 +1,44 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + } +} + +# Download trampoline resources. +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" + +# Use the trampoline script to run in docker. +build_file: "python-crc32c/.kokoro/trampoline.sh" + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-multi" +} +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-crc32c/.kokoro/release.sh" +} + +# Fetch the token needed for reporting release status to GitHub +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "yoshi-automation-github-key" + } + } +} + +# Fetch PyPI password +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "google-cloud-pypi-token-keystore-1" + } + } +} diff --git a/.kokoro/release/windows/windows_wheel.cfg b/.kokoro/release/windows/windows_wheel.cfg new file mode 100644 index 00000000..1f640148 --- /dev/null +++ b/.kokoro/release/windows/windows_wheel.cfg @@ -0,0 +1,14 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-crc32c/.kokoro/release-windows-wheel.sh" +} + +# Tell the trampoline which build file to use. +env_vars: { + key: "PACKAGE" + value: "crc32c" +} + +build_file: "python-crc32c/.kokoro/release-windows-wheel.bat" diff --git a/owlbot.py b/owlbot.py index 20587d18..cec3cf40 100644 --- a/owlbot.py +++ b/owlbot.py @@ -44,7 +44,8 @@ "setup.py", "setup.cfg", "testing/**/*", - ".kokoro/build.sh", + ".kokoro/release.sh", # not needed + ".kokoro/build.sh", # not needed ], ) diff --git a/scripts/manylinux/build.sh b/scripts/manylinux/build.sh index 516721fb..35fcc6aa 100755 --- a/scripts/manylinux/build.sh +++ b/scripts/manylinux/build.sh @@ -57,3 +57,7 @@ docker run \ --env BUILD_PYTHON=${BUILD_PYTHON} \ quay.io/pypa/manylinux2014_aarch64 \ /var/code/python-crc32c/scripts/manylinux/build_on_centos.sh + +if [[ "${PUBLISH_WHEELS}" == "true" ]]; then + . /${MANYLINUX_DIR}/publish_python_wheel.sh +fi diff --git a/.kokoro/release.sh b/scripts/manylinux/publish_python_wheel.sh similarity index 71% rename from .kokoro/release.sh rename to scripts/manylinux/publish_python_wheel.sh index ade25125..ccc55a4c 100755 --- a/.kokoro/release.sh +++ b/scripts/manylinux/publish_python_wheel.sh @@ -15,15 +15,15 @@ set -eo pipefail +python3.9 -m pip install "setuptools<71" + # Start the releasetool reporter -python3 -m pip install --require-hashes -r github/python-crc32c/.kokoro/requirements.txt -python3 -m releasetool publish-reporter-script > /tmp/publisher-script; source /tmp/publisher-script +python3.9 -m pip install --require-hashes -r ${REPO_ROOT}/.kokoro/requirements.txt +python3.9 -m releasetool publish-reporter-script > /tmp/publisher-script; source /tmp/publisher-script # Disable buffering, so that the logs stream through. export PYTHONUNBUFFERED=1 # Move into the package, build the distribution and upload. TWINE_PASSWORD=$(cat "${KOKORO_KEYSTORE_DIR}/73713_google-cloud-pypi-token-keystore-1") -cd github/python-crc32c -python3 setup.py sdist bdist_wheel -twine upload --username __token__ --password "${TWINE_PASSWORD}" dist/* +python3.9 -m twine upload --skip-existing --username __token__ --password "${TWINE_PASSWORD}" ${REPO_ROOT}/wheels/* diff --git a/scripts/osx/build_python_wheel.sh b/scripts/osx/build_python_wheel.sh index 26eb9fad..b00a8308 100755 --- a/scripts/osx/build_python_wheel.sh +++ b/scripts/osx/build_python_wheel.sh @@ -78,6 +78,10 @@ ${VENV}/bin/delocate-wheel \ --check-archs \ ${DIST_WHEELS}/google_crc32c*${PY_TAG}*.whl +if [[ "${PUBLISH_WHEELS}" == "true" ]]; then + . /${OSX_DIR}/publish_python_wheel.sh +fi + # Clean up. rm -fr ${DIST_WHEELS} rm -fr ${VENV} diff --git a/scripts/osx/publish_python_wheel.sh b/scripts/osx/publish_python_wheel.sh new file mode 100755 index 00000000..6fef9f3e --- /dev/null +++ b/scripts/osx/publish_python_wheel.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +PYTHON=$(PYENV_VERSION=3.9 pyenv which python) +PYTHON_BIN=$(dirname ${PYTHON}) + +# Start the releasetool reporter +${PYTHON} -m pip install --require-hashes -r ${REPO_ROOT}/.kokoro/requirements.txt +${PYTHON} -m releasetool publish-reporter-script > /tmp/publisher-script; source /tmp/publisher-script + +TWINE=${PYTHON_BIN}/twine + +# Disable buffering, so that the logs stream through. +export PYTHONUNBUFFERED=1 + +TWINE_PASSWORD=$(cat "${KOKORO_KEYSTORE_DIR}/73713_google-cloud-pypi-token-keystore-1") +${PYTHON} -m twine upload --skip-existing --username __token__ --password "${TWINE_PASSWORD}" ${REPO_ROOT}/dist_wheels/*