-
Notifications
You must be signed in to change notification settings - Fork 15.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[Kokoro/Bazel] Send Bazel build results to BES. (#10008)
This change adds bazel_wrapper.sh, which uses values set by Kokoro to log remotely.
- Loading branch information
Showing
5 changed files
with
133 additions
and
44 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,71 @@ | ||
#!/bin/bash | ||
|
||
# Wrapper for invoking bazel on Kokoro. | ||
# | ||
# This script adds extra flags to a bazel invocation when it is run from Kokoro. | ||
# When the special environment variables are not present (e.g., if you run | ||
# Kokoro build scripts locally), this script is equivalent to the "bazel" | ||
# command. | ||
# | ||
# Example of running directly: | ||
# path/to/bazel_wrapper.sh build //... | ||
# | ||
# Example of `source`ing: | ||
# source path/to/bazel_wrapper.sh | ||
# bazel_wrapper build //... | ||
|
||
function bazel_wrapper::gen_invocation_id() { | ||
# Create a new invocation ID and store in the artifacts dir. | ||
local _invocation_id=$(uuidgen | tr A-Z a-z) | ||
|
||
# Put the new invocation ID at the start of the output IDs file. Some | ||
# Google-internal tools only look at the first entry, so this ensures the most | ||
# recent entry is first. | ||
local _ids_file=${KOKORO_ARTIFACTS_DIR}/bazel_invocation_ids | ||
local _temp_ids=$(mktemp) | ||
echo ${_invocation_id} > ${_temp_ids} | ||
[[ -e ${_ids_file} ]] && cat ${_ids_file} >> ${_temp_ids} | ||
mv -f ${_temp_ids} ${_ids_file} | ||
|
||
echo -n ${_invocation_id} | ||
} | ||
|
||
# Prints flags to use on Kokoro. | ||
function bazel_wrapper::kokoro_flags() { | ||
[[ -n ${KOKORO_BES_PROJECT_ID:-} ]] || return | ||
|
||
local -a _flags | ||
_flags+=( | ||
--bes_backend=${KOKORO_BES_BACKEND_ADDRESS:-buildeventservice.googleapis.com} | ||
--bes_results_url=https://source.cloud.google.com/results/invocations/ | ||
--invocation_id=$(bazel_wrapper::gen_invocation_id) | ||
--project_id=${KOKORO_BES_PROJECT_ID} # --bes_instance_name in Bazel 5+ | ||
--remote_cache=https://storage.googleapis.com/protobuf-bazel-cache | ||
) | ||
if [[ -n ${KOKORO_BAZEL_AUTH_CREDENTIAL:-} ]]; then | ||
_flags+=( --google_credentials=${KOKORO_BAZEL_AUTH_CREDENTIAL} ) | ||
else | ||
_flags+=( --google_default_credentials=true ) | ||
fi | ||
|
||
echo "${_flags[@]}" | ||
} | ||
|
||
# Runs bazel with Kokoro flags, if appropriate. | ||
function bazel_wrapper() { | ||
local -a _flags | ||
|
||
# We might need to add flags. They need to come after any startup flags and | ||
# the command, but before any terminating "--", so copy them into the _flags | ||
# variable. | ||
until (( ${#@} == 0 )) || [[ $1 == "--" ]]; do | ||
_flags+=( "${1}" ); shift | ||
done | ||
|
||
# Set the `BAZEL` env variable to override the actual bazel binary to use: | ||
${BAZEL:=bazel} "${_flags[@]}" $(bazel_wrapper::kokoro_flags) "${@}" | ||
} | ||
|
||
# If this script was called directly, run bazel. Otherwise (i.e., this script | ||
# was `source`d), the sourcing script will call bazel_wrapper themselves. | ||
(( ${#BASH_SOURCE[@]} == 1 )) && bazel_wrapper "${@}" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,36 +1,44 @@ | ||
#!/bin/bash | ||
# | ||
# Build file to set up and run tests | ||
set -ex | ||
set -eu | ||
|
||
# Install Bazel 4.0.0. | ||
use_bazel.sh 4.0.0 | ||
bazel version | ||
|
||
# Change to repo root | ||
cd $(dirname $0)/../../.. | ||
source kokoro/common/pyenv.sh | ||
|
||
git submodule update --init --recursive | ||
# Get kokoro scripts from repo root by default. | ||
: ${SCRIPT_ROOT:=$(pwd)} | ||
source ${SCRIPT_ROOT}/kokoro/common/pyenv.sh | ||
|
||
# Disabled for now, re-enable if appropriate. | ||
# //:build_files_updated_unittest \ | ||
|
||
bazel test \ | ||
-k --copt=-Werror --host_copt=-Werror --test_output=errors \ | ||
//build_defs:all \ | ||
//java:tests \ | ||
//src/... \ | ||
//:protobuf_python \ | ||
bazel_args=( | ||
test | ||
--keep_going | ||
--copt=-Werror | ||
--host_copt=-Werror | ||
--test_output=errors | ||
-- | ||
//... | ||
-//objectivec/... # only works on macOS | ||
@com_google_protobuf_examples//... | ||
) | ||
|
||
${SCRIPT_ROOT}/kokoro/common/bazel_wrapper.sh "${bazel_args[@]}" | ||
|
||
# Verify that we can build successfully from generated tar files. | ||
( | ||
pyenv versions | ||
pyenv shell 2.7.9 # python2 required for old googletest autotools support | ||
git submodule update --init --recursive | ||
./autogen.sh && ./configure && make -j$(nproc) dist | ||
) | ||
DIST=`ls *.tar.gz` | ||
tar -xf $DIST | ||
cd ${DIST//.tar.gz} | ||
bazel build //:protobuf //:protobuf_java | ||
${SCRIPT_ROOT}/kokoro/common/bazel_wrapper.sh build //:protobuf //:protobuf_java |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,9 @@ | ||
# Common config shared by presubmit and continuous. | ||
|
||
bazel_setting: { | ||
project_id: "protobuf-build" | ||
bes_backend_address: "buildeventservice.googleapis.com" | ||
foundry_backend_address: "remotebuildexecution.googleapis.com" | ||
upsalite_frontend_address: "https://source.cloud.google.com" | ||
local_execution: true | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,9 @@ | ||
# Common config shared by presubmit and continuous. | ||
|
||
bazel_setting: { | ||
project_id: "protobuf-build" | ||
bes_backend_address: "buildeventservice.googleapis.com" | ||
foundry_backend_address: "remotebuildexecution.googleapis.com" | ||
upsalite_frontend_address: "https://source.cloud.google.com" | ||
local_execution: true | ||
} |