build: add release scripts using kokoro (#236)

Co-authored-by: @vishwarajanand <vishwarajanand@users.noreply.github.com>
Co-authored-by: @andrewsg <andrewsg@users.noreply.github.com>

.kokoro/release-manylinux-wheel.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -eo pipefail
+
+cd github/python-crc32c
+
+export PUBLISH_WHEELS="true"
+
+# Build and publish ManyLinux wheels
+./scripts/manylinux/build.sh

.kokoro/release-osx-wheel.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -eo pipefail
+
+REPO_ROOT=$(pwd)/github/python-crc32c
+cd github/python-crc32c
+
+export PUBLISH_WHEELS="true"
+
+# Build and publish OSX wheels
+./scripts/osx/build.sh

.kokoro/release-windows-wheel.bat

@@ -0,0 +1,44 @@
+@rem Copyright 2019 Google LLC. All rights reserved.
+@rem
+@rem Licensed under the Apache License, Version 2.0 (the "License");
+@rem you may not use this file except in compliance with the License.
+@rem You may obtain a copy of the License at
+@rem
+@rem     http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+
+@echo "Starting Windows release"
+
+cd /d %~dp0
+cd ..
+
+@echo "Build Wheel"
+call scripts\windows\build.bat || goto :error
+
+@echo "Start the releasetool reporter"
+call py -3 -m pip install gcp-releasetool || goto :error
+call py -3 -m releasetool publish-reporter-script > C:\temp\publisher-script || goto :error
+
+@echo "Ensure that we have the latest versions of Twine, Wheel, and Setuptools."
+call py -3 -m pip install --upgrade twine wheel setuptools || goto :error
+
+@echo "Disable buffering, so that the logs stream through."
+set PYTHONUNBUFFERED=1
+
+@echo "## RELASE WORKFLOW SUCCESSFUL ##"
+@echo "## Uploading Wheels ##"
+
+@echo "Move into the package, build the distribution and upload."
+set /p TWINE_PASSWORD=<%KOKORO_KEYSTORE_DIR%/73713_google-cloud-pypi-token-keystore-1
+call py -3 setup.py sdist || goto :error
+call py -3 -m twine upload --skip-existing --username __token__ --password "%TWINE_PASSWORD%" dist/* wheels/google_crc32c* || goto :error
+
+goto :EOF
+
+:error
+exit /b 1

.kokoro/release/linux/common.cfg

@@ -0,0 +1,44 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Build logs will be here
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.xml"
+  }
+}
+
+# Download trampoline resources.
+gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline"
+
+# Use the trampoline script to run in docker.
+build_file: "python-crc32c/.kokoro/trampoline.sh"
+
+# Configure the docker image for kokoro-trampoline.
+env_vars: {
+    key: "TRAMPOLINE_IMAGE"
+    value: "gcr.io/cloud-devrel-kokoro-resources/python-multi"
+}
+env_vars: {
+    key: "TRAMPOLINE_BUILD_FILE"
+    value: "github/python-crc32c/.kokoro/release.sh"
+}
+
+# Fetch the token needed for reporting release status to GitHub
+before_action {
+  fetch_keystore {
+    keystore_resource {
+      keystore_config_id: 73713
+      keyname: "yoshi-automation-github-key"
+    }
+  }
+}
+
+# Fetch PyPI password
+before_action {
+  fetch_keystore {
+    keystore_resource {
+      keystore_config_id: 73713
+      keyname: "google-cloud-pypi-token-keystore-1"
+    }
+  }
+}

.kokoro/release/linux/linux_wheel.cfg

@@ -0,0 +1,14 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+env_vars: {
+    key: "TRAMPOLINE_BUILD_FILE"
+    value: "github/python-crc32c/.kokoro/release-manylinux-wheel.sh"
+}
+
+# Tell the trampoline which build file to use.
+env_vars: {
+    key: "PACKAGE"
+    value: "crc32c"
+}
+
+build_file: "python-crc32c/.kokoro/release-manylinux-wheel.sh"

.kokoro/release/osx/common.cfg

@@ -0,0 +1,44 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Build logs will be here
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.xml"
+  }
+}
+
+# Download trampoline resources.
+gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline"
+
+# Use the trampoline script to run in docker.
+build_file: "python-crc32c/.kokoro/trampoline.sh"
+
+# Configure the docker image for kokoro-trampoline.
+env_vars: {
+    key: "TRAMPOLINE_IMAGE"
+    value: "gcr.io/cloud-devrel-kokoro-resources/python-multi"
+}
+env_vars: {
+    key: "TRAMPOLINE_BUILD_FILE"
+    value: "github/python-crc32c/.kokoro/release.sh"
+}
+
+# Fetch the token needed for reporting release status to GitHub
+before_action {
+  fetch_keystore {
+    keystore_resource {
+      keystore_config_id: 73713
+      keyname: "yoshi-automation-github-key"
+    }
+  }
+}
+
+# Fetch PyPI password
+before_action {
+  fetch_keystore {
+    keystore_resource {
+      keystore_config_id: 73713
+      keyname: "google-cloud-pypi-token-keystore-1"
+    }
+  }
+}

.kokoro/release/osx/osx-slcn_wheel.cfg

@@ -0,0 +1,14 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+env_vars: {
+    key: "TRAMPOLINE_BUILD_FILE"
+    value: "github/python-crc32c/.kokoro/release-osx-wheel.sh"
+}
+
+# Tell the trampoline which build file to use.
+env_vars: {
+    key: "PACKAGE"
+    value: "crc32c"
+}
+
+build_file: "python-crc32c/.kokoro/release-osx-wheel.sh"

.kokoro/release/osx/osx_wheel.cfg

@@ -0,0 +1,14 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+env_vars: {
+    key: "TRAMPOLINE_BUILD_FILE"
+    value: "github/python-crc32c/.kokoro/release-osx-wheel.sh"
+}
+
+# Tell the trampoline which build file to use.
+env_vars: {
+    key: "PACKAGE"
+    value: "crc32c"
+}
+
+build_file: "python-crc32c/.kokoro/release-osx-wheel.sh"

.kokoro/release/windows/common.cfg

@@ -0,0 +1,44 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Build logs will be here
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.xml"
+  }
+}
+
+# Download trampoline resources.
+gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline"
+
+# Use the trampoline script to run in docker.
+build_file: "python-crc32c/.kokoro/trampoline.sh"
+
+# Configure the docker image for kokoro-trampoline.
+env_vars: {
+    key: "TRAMPOLINE_IMAGE"
+    value: "gcr.io/cloud-devrel-kokoro-resources/python-multi"
+}
+env_vars: {
+    key: "TRAMPOLINE_BUILD_FILE"
+    value: "github/python-crc32c/.kokoro/release.sh"
+}
+
+# Fetch the token needed for reporting release status to GitHub
+before_action {
+  fetch_keystore {
+    keystore_resource {
+      keystore_config_id: 73713
+      keyname: "yoshi-automation-github-key"
+    }
+  }
+}
+
+# Fetch PyPI password
+before_action {
+  fetch_keystore {
+    keystore_resource {
+      keystore_config_id: 73713
+      keyname: "google-cloud-pypi-token-keystore-1"
+    }
+  }
+}

.kokoro/release/windows/windows_wheel.cfg

@@ -0,0 +1,14 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+env_vars: {
+    key: "TRAMPOLINE_BUILD_FILE"
+    value: "github/python-crc32c/.kokoro/release-windows-wheel.sh"
+}
+
+# Tell the trampoline which build file to use.
+env_vars: {
+    key: "PACKAGE"
+    value: "crc32c"
+}
+
+build_file: "python-crc32c/.kokoro/release-windows-wheel.bat"

owlbot.py

@@ -44,7 +44,8 @@
         "setup.py",
         "setup.cfg",
         "testing/**/*",
-        ".kokoro/build.sh",
+        ".kokoro/release.sh", # not needed
+        ".kokoro/build.sh", # not needed
     ],
 )
 

scripts/manylinux/build.sh

@@ -57,3 +57,7 @@ docker run \
     --env BUILD_PYTHON=${BUILD_PYTHON} \
     quay.io/pypa/manylinux2014_aarch64 \
     /var/code/python-crc32c/scripts/manylinux/build_on_centos.sh
+
+if [[ "${PUBLISH_WHEELS}" == "true" ]]; then
+    . /${MANYLINUX_DIR}/publish_python_wheel.sh
+fi

.kokoro/release.sh → scripts/manylinux/publish_python_wheel.sh

@@ -15,15 +15,15 @@
 
 set -eo pipefail
 
+python3.9 -m pip install "setuptools<71"
+
 # Start the releasetool reporter
-python3 -m pip install --require-hashes -r github/python-crc32c/.kokoro/requirements.txt
-python3 -m releasetool publish-reporter-script > /tmp/publisher-script; source /tmp/publisher-script
+python3.9 -m pip install --require-hashes -r ${REPO_ROOT}/.kokoro/requirements.txt
+python3.9 -m releasetool publish-reporter-script > /tmp/publisher-script; source /tmp/publisher-script
 
 # Disable buffering, so that the logs stream through.
 export PYTHONUNBUFFERED=1
 
 # Move into the package, build the distribution and upload.
 TWINE_PASSWORD=$(cat "${KOKORO_KEYSTORE_DIR}/73713_google-cloud-pypi-token-keystore-1")
-cd github/python-crc32c
-python3 setup.py sdist bdist_wheel
-twine upload --username __token__ --password "${TWINE_PASSWORD}" dist/*
+python3.9 -m twine upload --skip-existing --username __token__ --password "${TWINE_PASSWORD}" ${REPO_ROOT}/wheels/*

scripts/osx/build_python_wheel.sh

@@ -78,6 +78,10 @@ ${VENV}/bin/delocate-wheel \
     --check-archs \
     ${DIST_WHEELS}/google_crc32c*${PY_TAG}*.whl
 
+if [[ "${PUBLISH_WHEELS}" == "true" ]]; then
+    . /${OSX_DIR}/publish_python_wheel.sh
+fi
+
 # Clean up.
 rm -fr ${DIST_WHEELS}
 rm -fr ${VENV}

scripts/osx/publish_python_wheel.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eo pipefail
+
+PYTHON=$(PYENV_VERSION=3.9 pyenv which python)
+PYTHON_BIN=$(dirname ${PYTHON})
+
+# Start the releasetool reporter
+${PYTHON} -m pip install --require-hashes -r ${REPO_ROOT}/.kokoro/requirements.txt
+${PYTHON} -m releasetool publish-reporter-script > /tmp/publisher-script; source /tmp/publisher-script
+
+TWINE=${PYTHON_BIN}/twine
+
+# Disable buffering, so that the logs stream through.
+export PYTHONUNBUFFERED=1
+
+TWINE_PASSWORD=$(cat "${KOKORO_KEYSTORE_DIR}/73713_google-cloud-pypi-token-keystore-1")
+${PYTHON} -m twine upload --skip-existing --username __token__ --password "${TWINE_PASSWORD}" ${REPO_ROOT}/dist_wheels/*