Add remote caching to bazel builds (#10767)

* Add remote cache to linux builds

* Remove BES flags

* Remove BES flags from the right file

* Migrate all Bazel kokoro builds to use remote caching

* Remove BES logic

* Fix mac ruby tests

* Give mac/windows builds GCP access

* Adding quotes to prevent issues with common flags

* Adding command echoing in windows builds

* Try enabling command echoing again

* Adding invocation id for windows bazel build

* Third try

* Adding credentials to windows build

kokoro/common/bazel_flags.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Helper for setting up common bazel flags in Kokoro.
+#
+# This script prints extra flags to a bazel invocation when it is run from
+# Kokoro.  When the special environment variables are not present (e.g., if you
+# run Kokoro build scripts locally), this script only flips some debug settings.
+#
+# Example of running directly:
+#   bazel test $(path/to/bazel_flags.sh) //...
+
+function bazel_flags::gen_invocation_id() {
+  # Create a new invocation ID and store in the artifacts dir.
+  local _invocation_id=$(uuidgen | tr A-Z a-z)
+
+  # Put the new invocation ID at the start of the output IDs file. Some
+  # Google-internal tools only look at the first entry, so this ensures the most
+  # recent entry is first.
+  local _ids_file=${KOKORO_ARTIFACTS_DIR}/bazel_invocation_ids
+  local _temp_ids=$(mktemp)
+  echo ${_invocation_id} > ${_temp_ids}
+  [[ -e ${_ids_file} ]] && cat ${_ids_file} >> ${_temp_ids}
+  mv -f ${_temp_ids} ${_ids_file}
+
+  echo -n ${_invocation_id}
+}
+
+# Prints flags to use on Kokoro.
+function bazel_flags::kokoro_flags() {
+  [[ -n ${KOKORO_JOB_NAME:-} ]] || return
+
+  local -a _flags
+  _flags+=(
+    --invocation_id=$(bazel_flags::gen_invocation_id)
+    --remote_cache=https://storage.googleapis.com/protobuf-bazel-cache/${KOKORO_JOB_NAME}
+  )
+  if [[ -n ${KOKORO_BAZEL_AUTH_CREDENTIAL:-} ]]; then
+    _flags+=( --google_credentials=${KOKORO_BAZEL_AUTH_CREDENTIAL} )
+  else
+    _flags+=( --google_default_credentials=true )
+  fi
+
+  echo "${_flags[@]}"
+}
+
+echo "$(bazel_flags::kokoro_flags) --keep_going --test_output=errors"

kokoro/linux/bazel.sh

@@ -39,8 +39,7 @@ function run {
     -v $GIT_REPO_ROOT:/workspace \
     $CONTAINER_IMAGE \
     test \
-    --keep_going \
-    --test_output=streamed \
+    $(${GIT_REPO_ROOT}/kokoro/common/bazel_flags.sh) \
     ${ENVS[@]} \
     $PLATFORM_CONFIG \
     $BAZEL_CONFIG \

kokoro/macos/cpp/build.sh

@@ -14,4 +14,4 @@ source kokoro/macos/prepare_build_macos_rc
 #
 # Run build
 #
-bazel test //src/... -k --test_output=streamed
+bazel test $(kokoro/common/bazel_flags.sh) //src/...

kokoro/macos/cpp/common.cfg

@@ -3,3 +3,25 @@
 # Location of the build script in repository
 build_file: "protobuf/kokoro/macos/cpp/build.sh"
 timeout_mins: 1440
+
+before_action {
+  fetch_keystore {
+    keystore_resource {
+      keystore_config_id: 77103
+      keyname: "kokoro_gcp_service"
+    }
+  }
+}
+bazel_setting {
+  project_id: "protobuf-build"
+  bes_backend_address: "buildeventservice.googleapis.com"
+  foundry_backend_address: "remotebuildexecution.googleapis.com"
+  upsalite_frontend_address: "https://source.cloud.google.com"
+  local_execution: true
+
+  # Need to be same as the fetch_keystore entry in the previous step.
+  auth_credential: {
+    keystore_config_id: 77103
+    keyname: "kokoro_gcp_service"
+  }
+}

kokoro/macos/objectivec_ios_debug/common.cfg

@@ -3,3 +3,25 @@
 # Location of the build script in repository
 build_file: "protobuf/kokoro/macos/objectivec_ios_debug/build.sh"
 timeout_mins: 1440
+
+before_action {
+  fetch_keystore {
+    keystore_resource {
+      keystore_config_id: 77103
+      keyname: "kokoro_gcp_service"
+    }
+  }
+}
+bazel_setting {
+  project_id: "protobuf-build"
+  bes_backend_address: "buildeventservice.googleapis.com"
+  foundry_backend_address: "remotebuildexecution.googleapis.com"
+  upsalite_frontend_address: "https://source.cloud.google.com"
+  local_execution: true
+
+  # Need to be same as the fetch_keystore entry in the previous step.
+  auth_credential: {
+    keystore_config_id: 77103
+    keyname: "kokoro_gcp_service"
+  }
+}

kokoro/macos/objectivec_ios_release/common.cfg

@@ -3,3 +3,25 @@
 # Location of the build script in repository
 build_file: "protobuf/kokoro/macos/objectivec_ios_release/build.sh"
 timeout_mins: 1440
+
+before_action {
+  fetch_keystore {
+    keystore_resource {
+      keystore_config_id: 77103
+      keyname: "kokoro_gcp_service"
+    }
+  }
+}
+bazel_setting {
+  project_id: "protobuf-build"
+  bes_backend_address: "buildeventservice.googleapis.com"
+  foundry_backend_address: "remotebuildexecution.googleapis.com"
+  upsalite_frontend_address: "https://source.cloud.google.com"
+  local_execution: true
+
+  # Need to be same as the fetch_keystore entry in the previous step.
+  auth_credential: {
+    keystore_config_id: 77103
+    keyname: "kokoro_gcp_service"
+  }
+}

kokoro/macos/objectivec_osx/common.cfg

@@ -3,3 +3,25 @@
 # Location of the build script in repository
 build_file: "protobuf/kokoro/macos/objectivec_osx/build.sh"
 timeout_mins: 1440
+
+before_action {
+  fetch_keystore {
+    keystore_resource {
+      keystore_config_id: 77103
+      keyname: "kokoro_gcp_service"
+    }
+  }
+}
+bazel_setting {
+  project_id: "protobuf-build"
+  bes_backend_address: "buildeventservice.googleapis.com"
+  foundry_backend_address: "remotebuildexecution.googleapis.com"
+  upsalite_frontend_address: "https://source.cloud.google.com"
+  local_execution: true
+
+  # Need to be same as the fetch_keystore entry in the previous step.
+  auth_credential: {
+    keystore_config_id: 77103
+    keyname: "kokoro_gcp_service"
+  }
+}

kokoro/macos/php74/common.cfg

@@ -3,3 +3,25 @@
 # Location of the build script in repository
 build_file: "protobuf/kokoro/macos/php74/build.sh"
 timeout_mins: 1440
+
+before_action {
+  fetch_keystore {
+    keystore_resource {
+      keystore_config_id: 77103
+      keyname: "kokoro_gcp_service"
+    }
+  }
+}
+bazel_setting {
+  project_id: "protobuf-build"
+  bes_backend_address: "buildeventservice.googleapis.com"
+  foundry_backend_address: "remotebuildexecution.googleapis.com"
+  upsalite_frontend_address: "https://source.cloud.google.com"
+  local_execution: true
+
+  # Need to be same as the fetch_keystore entry in the previous step.
+  auth_credential: {
+    keystore_config_id: 77103
+    keyname: "kokoro_gcp_service"
+  }
+}

kokoro/macos/php80/common.cfg

@@ -3,3 +3,25 @@
 # Location of the build script in repository
 build_file: "protobuf/kokoro/macos/php80/build.sh"
 timeout_mins: 1440
+
+before_action {
+  fetch_keystore {
+    keystore_resource {
+      keystore_config_id: 77103
+      keyname: "kokoro_gcp_service"
+    }
+  }
+}
+bazel_setting {
+  project_id: "protobuf-build"
+  bes_backend_address: "buildeventservice.googleapis.com"
+  foundry_backend_address: "remotebuildexecution.googleapis.com"
+  upsalite_frontend_address: "https://source.cloud.google.com"
+  local_execution: true
+
+  # Need to be same as the fetch_keystore entry in the previous step.
+  auth_credential: {
+    keystore_config_id: 77103
+    keyname: "kokoro_gcp_service"
+  }
+}

kokoro/macos/python/build.sh

@@ -9,4 +9,5 @@ cd $(dirname $0)/../../..
 KOKORO_INSTALL_VENV=yes
 source kokoro/macos/prepare_build_macos_rc
 
-bazel test //python/... @upb//python/... -k --macos_minimum_os=10.9 --test_output=streamed
+bazel test //python/... @upb//python/...  $(kokoro/common/bazel_flags.sh) \
+  --macos_minimum_os=10.9

kokoro/macos/python/common.cfg

@@ -9,3 +9,25 @@ action {
     regex: "**/*"
   }
 }
+
+before_action {
+  fetch_keystore {
+    keystore_resource {
+      keystore_config_id: 77103
+      keyname: "kokoro_gcp_service"
+    }
+  }
+}
+bazel_setting {
+  project_id: "protobuf-build"
+  bes_backend_address: "buildeventservice.googleapis.com"
+  foundry_backend_address: "remotebuildexecution.googleapis.com"
+  upsalite_frontend_address: "https://source.cloud.google.com"
+  local_execution: true
+
+  # Need to be same as the fetch_keystore entry in the previous step.
+  auth_credential: {
+    keystore_config_id: 77103
+    keyname: "kokoro_gcp_service"
+  }
+}

kokoro/macos/python_cpp/build.sh

@@ -9,4 +9,5 @@ cd $(dirname $0)/../../..
 KOKORO_INSTALL_VENV=yes
 source kokoro/macos/prepare_build_macos_rc
 
-bazel test //python/... -k --macos_minimum_os=10.9 --test_output=streamed --define=use_fast_cpp_protos=true
+bazel test //python/...  $(kokoro/common/bazel_flags.sh) \
+  --macos_minimum_os=10.9 --define=use_fast_cpp_protos=true

kokoro/macos/python_cpp/common.cfg

@@ -3,3 +3,25 @@
 # Location of the build script in repository
 build_file: "protobuf/kokoro/macos/python_cpp/build.sh"
 timeout_mins: 1440
+
+before_action {
+  fetch_keystore {
+    keystore_resource {
+      keystore_config_id: 77103
+      keyname: "kokoro_gcp_service"
+    }
+  }
+}
+bazel_setting {
+  project_id: "protobuf-build"
+  bes_backend_address: "buildeventservice.googleapis.com"
+  foundry_backend_address: "remotebuildexecution.googleapis.com"
+  upsalite_frontend_address: "https://source.cloud.google.com"
+  local_execution: true
+
+  # Need to be same as the fetch_keystore entry in the previous step.
+  auth_credential: {
+    keystore_config_id: 77103
+    keyname: "kokoro_gcp_service"
+  }
+}

kokoro/macos/ruby25/common.cfg

@@ -3,3 +3,25 @@
 # Location of the build script in repository
 build_file: "protobuf/kokoro/macos/ruby25/build.sh"
 timeout_mins: 1440
+
+before_action {
+  fetch_keystore {
+    keystore_resource {
+      keystore_config_id: 77103
+      keyname: "kokoro_gcp_service"
+    }
+  }
+}
+bazel_setting {
+  project_id: "protobuf-build"
+  bes_backend_address: "buildeventservice.googleapis.com"
+  foundry_backend_address: "remotebuildexecution.googleapis.com"
+  upsalite_frontend_address: "https://source.cloud.google.com"
+  local_execution: true
+
+  # Need to be same as the fetch_keystore entry in the previous step.
+  auth_credential: {
+    keystore_config_id: 77103
+    keyname: "kokoro_gcp_service"
+  }
+}