chore: clean up hermetic build script and test (#1943)

* feat: add

* generate gapic and proto folder

* refactor utilities

* add an action to verify

* checkout googleapis-gen

* setup repo name

* add commit hash of googleapis-gen

* change secret

* change token

* change to git clone

* change user name

* add input list

* include resources folder in main

* remove grpc version in `*ServiceGrpc.java`

* change destination path

* compare generation result with googleapis-gen

* fix type in diff command

* checkout repo using checkout action

* checkout repos as nested repo

* sparse checkout googleapis

* Revert "sparse checkout googleapis"

This reverts commit 3d612f8.

* change library

* change step name

* add a readme

* make grpc version optional

* make protobuf version optional

* checkout master branch, rather than a commit hash

* allow snapshot version of generator

* download snapshot of generator parent pom

* update README

* download generator and grpc using mvn

* change error message

* add maven central mirror

* add comments in utilities

* add comments

* add an integration test

* fail fast if no file is found

* do not delete google/

* get protobuf version from WORKSPACE

* add instructions on download `google/` from googleapis

* add comments

* update description of `destination_path`

* update comments

* download dependencies using `curl`

* increase download time

* remove comment

* add samples directory in readme

* remove prerequisite about `proto_path`

* add explanation in prerequisite

* add example to generate showcase

* chore: clean up hermetic build script and test

Allow running on OSX in addition to Linux.

* fix exit code

* fix sed

* fix rm command

* run multiple os

* change cpu arch for macos

* change os arch

* change tag to commit hash

* remove 3p action

* change inputs

---------

Co-authored-by: JoeWang1127 <joewa@google.com>

.github/workflows/verify_library_generation.yaml

@@ -10,10 +10,11 @@ on:
 name: verify_library_generation_against_googleapis-gen
 jobs:
   verify_library_generation:
-    runs-on: ubuntu-22.04
     strategy:
       matrix:
         java: [ 8 ]
+        os: [ ubuntu-22.04, macos-12 ]
+    runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v3
     - uses: actions/setup-java@v3
@@ -27,4 +28,5 @@ jobs:
         library_generation/generate_library_integration_test.sh \
         -p google/bigtable/v2 \
         -d google-cloud-bigtable-v2-java \
-        --googleapis_gen_url https://cloud-java-bot:${{ secrets.CLOUD_JAVA_BOT_GITHUB_TOKEN }}@github.com/googleapis/googleapis-gen.git
+        --googleapis_gen_url https://cloud-java-bot:${{ secrets.CLOUD_JAVA_BOT_GITHUB_TOKEN }}@github.com/googleapis/googleapis-gen.git \
+        --os_type ${{ matrix.os }}

library_generation/README.md

@@ -108,6 +108,10 @@ The default value is `true`.
 
 Use `--include_samples` to specify the value.
 
+### os_architecture (optional)
+Choose the protoc binary type from https://github.com/protocolbuffers/protobuf/releases.
+Default is "linux-x86_64".
+
 ## An example to generate a client library
 ```
 library_generation/generate_library.sh \

library_generation/gapic-generator-java-wrapper

@@ -3,5 +3,4 @@
 set -e
 
 # Wrap gapic-generator-java.jar because protoc requires the plugin to be executable.
-working_directory=$(dirname "$(readlink -f "$0")")
-exec java -classpath "$working_directory/gapic-generator-java-$gapic_generator_version.jar" com.google.api.generator.Main
+exec java -classpath "gapic-generator-java-$gapic_generator_version.jar" com.google.api.generator.Main

library_generation/generate_library.sh

@@ -41,6 +41,10 @@ case $key in
     include_samples="$2"
     shift
     ;;
+    --os_architecture)
+    os_architecture="$2"
+    shift
+    ;;
     *)
     echo "Invalid option: [$1]"
     exit 1
@@ -49,10 +53,9 @@ esac
 shift # past argument or value
 done
 
-working_directory=$(dirname "$(readlink -f "$0")")
+script_dir=$(dirname "$(readlink -f "$0")")
 # source utility functions
-cd "$working_directory"
-source ./utilities.sh
+source "$script_dir"/utilities.sh
 
 if [ -z "$protobuf_version" ]; then
   protobuf_version=$(get_protobuf_version "$gapic_generator_version")
@@ -74,25 +77,25 @@ if [ -z "$include_samples" ]; then
   include_samples="true"
 fi
 
-cd "$working_directory"
+if [ -z "$os_architecture" ]; then
+  os_architecture="linux-x86_64"
+fi
+
 mkdir -p "$destination_path"
-destination_path="$working_directory/$destination_path"
 ##################### Section 0 #####################
 # prepare tooling
 #####################################################
-cd "$working_directory"
 # the order of services entries in gapic_metadata.json is relevant to the
 # order of proto file, sort the proto files with respect to their name to
 # get a fixed order.
 proto_files=$(find "$proto_path" -type f  -name "*.proto" | sort)
 folder_name=$(extract_folder_name "$destination_path")
 # download gapic-generator-java, protobuf and grpc plugin.
-download_tools "$gapic_generator_version" "$protobuf_version" "$grpc_version"
+download_tools "$gapic_generator_version" "$protobuf_version" "$grpc_version" "$os_architecture"
 ##################### Section 1 #####################
 # generate grpc-*/
 #####################################################
-cd "$working_directory"
-"$protoc_path"/protoc "--plugin=protoc-gen-rpc-plugin=$working_directory/protoc-gen-grpc-java-$grpc_version-linux-x86_64.exe" \
+"$protoc_path"/protoc "--plugin=protoc-gen-rpc-plugin=protoc-gen-grpc-java-$grpc_version-${os_architecture}.exe" \
 "--rpc-plugin_out=:$destination_path/java_grpc.jar" \
 $proto_files
 # unzip java_grpc.jar to grpc-*/src/main/java
@@ -104,9 +107,8 @@ remove_grpc_version
 ###################### Section 2 #####################
 ## generate gapic-*/, part of proto-*/, samples/
 ######################################################
-cd "$working_directory"
 "$protoc_path"/protoc --experimental_allow_proto3_optional \
-"--plugin=protoc-gen-java_gapic=$working_directory/gapic-generator-java-wrapper" \
+"--plugin=protoc-gen-java_gapic=$script_dir/gapic-generator-java-wrapper" \
 "--java_gapic_out=metadata:$destination_path/java_gapic_srcjar_raw.srcjar.zip" \
 "--java_gapic_opt=$(get_gapic_opts)" \
 ${proto_files} $(search_additional_protos)
@@ -125,7 +127,6 @@ if [ ! -d "$proto_dir" ]; then
   touch "$proto_dir"/PlaceholderFile.java
 fi
 
-cd "$working_directory"
 # move java_gapic_srcjar/src/main to gapic-*/src.
 mv_src_files "gapic" "main"
 # remove empty files in gapic-*/src/main/java
@@ -139,7 +140,6 @@ fi
 ##################### Section 3 #####################
 # generate proto-*/
 #####################################################
-cd "$working_directory"
 "$protoc_path"/protoc "--java_out=$destination_path/java_proto.jar" $proto_files
 # move java_gapic_srcjar/proto/src/main/java (generated resource name helper class)
 # to proto-*/src/main
@@ -151,10 +151,10 @@ remove_empty_files "proto"
 # copy proto files to proto-*/src/main/proto
 for proto_src in $proto_files; do
     mkdir -p "$destination_path/proto-$folder_name/src/main/proto"
-    cp -f --parents "$proto_src" "$destination_path/proto-$folder_name/src/main/proto"
+    rsync -R "$proto_src" "$destination_path/proto-$folder_name/src/main/proto"
 done
 ##################### Section 4 #####################
 # rm tar files
 #####################################################
 cd "$destination_path"
-rm -rf java_gapic_srcjar java_gapic_srcjar_raw.srcjar.zip java_grpc.jar java_proto.jar temp-codegen.srcjar
+rm -rf java_gapic_srcjar java_gapic_srcjar_raw.srcjar.zip java_grpc.jar java_proto.jar temp-codegen.srcjar

library_generation/generate_library_integration_test.sh

@@ -11,6 +11,10 @@ set -xeo pipefail
 #    `generate_library.sh`. GAPIC options to generate a library will be parsed
 #    from proto_path/BUILD.bazel.
 # 4. checkout the master branch googleapis-gen repository and compare the result.
+
+# defaults
+googleapis_gen_url="git@github.com:googleapis/googleapis-gen.git"
+
 while [[ $# -gt 0 ]]
 do
 key="$1"
@@ -27,6 +31,10 @@ case $key in
     googleapis_gen_url="$2"
     shift
     ;;
+    --os_type)
+    os_type="$2"
+    shift
+    ;;
     *)
     echo "Invalid option: [$1]"
     exit 1
@@ -43,14 +51,23 @@ get_version_from_WORKSPACE() {
   echo "$version"
 }
 
-working_directory=$(dirname "$(readlink -f "$0")")
-cd "$working_directory"
+sparse_clone() {
+  repo_url=$1
+  paths=$2
+  clone_dir=$(basename "${repo_url%.*}")
+  rm -rf "$clone_dir"
+  git clone -n --depth=1 --filter=tree:0 "$repo_url"
+  cd "$clone_dir"
+  git sparse-checkout set --no-cone $paths
+  git checkout
+  cd ..
+}
+
+script_dir=$(dirname "$(readlink -f "$0")")
 # checkout the master branch of googleapis/google (proto files) and WORKSPACE
 echo "Checking out googlapis repository..."
-git clone --branch=master --depth 1 -q https://github.com/googleapis/googleapis.git
-cp -r googleapis/google .
-cp googleapis/WORKSPACE .
-rm -rf googleapis
+sparse_clone https://github.com/googleapis/googleapis.git "$proto_path WORKSPACE google/api google/rpc google/cloud/common_resources.proto"
+cd googleapis
 # parse version of gapic-generator-java, protobuf and grpc from WORKSPACE
 gapic_generator_version=$(get_version_from_WORKSPACE "_gapic_generator_java_version" WORKSPACE "=")
 echo "The version of gapic-generator-java is $gapic_generator_version."
@@ -59,7 +76,6 @@ echo "The version of protobuf is $protobuf_version"
 grpc_version=$(get_version_from_WORKSPACE "_grpc_version" WORKSPACE "=")
 echo "The version of protoc-gen-grpc-java plugin is $gapic_generator_version."
 # parse GAPIC options from proto_path/BUILD.bazel
-cd $"$working_directory"
 transport="grpc"
 if grep -A 15 "java_gapic_library(" "$proto_path/BUILD.bazel" | grep -q "grpc+rest"; then
   transport="grpc+rest"
@@ -73,26 +89,40 @@ if grep -A 15 "java_gapic_assembly_gradle_pkg(" "$proto_path/BUILD.bazel" | grep
   include_samples="true"
 fi
 echo "GAPIC options are transport=$transport, rest_numeric_enums=$rest_numeric_enums, include_samples=$include_samples."
+os_architecture="linux-x86_64"
+if [[ "$os_type" == *"macos"* ]]; then
+ os_architecture="osx-x86_64"
+fi
+echo "OS Architecture is $os_architecture."
 # generate GAPIC client library
 echo "Generating library from $proto_path, to $destination_path..."
-"$working_directory"/generate_library.sh \
+"$script_dir"/generate_library.sh \
 -p "$proto_path" \
 -d "$destination_path" \
 --gapic_generator_version "$gapic_generator_version" \
 --protobuf_version "$protobuf_version" \
 --grpc_version "$grpc_version" \
 --transport "$transport" \
 --rest_numeric_enums "$rest_numeric_enums" \
---include_samples "$include_samples"
+--include_samples "$include_samples" \
+--os_architecture "$os_architecture"
 
 echo "Generate library finished."
 echo "Checking out googleapis-gen repository..."
-git clone --branch=master --depth 1 -q "$googleapis_gen_url"
+
+sparse_clone "$googleapis_gen_url" "$proto_path"
 
 echo "Compare generation result..."
-cd "$working_directory"
-diff -r "googleapis-gen/$proto_path/$destination_path" "$destination_path" -x "*gradle*"
-echo "Comparison finished, no difference is found."
-# clean up
-cd "$working_directory"
-rm -rf WORKSPACE googleapis-gen "$destination_path"
+RESULT=0
+diff -r "googleapis-gen/$proto_path/$destination_path" "$destination_path" -x "*gradle*" || RESULT=$?
+
+if [ $RESULT == 0 ] ; then
+ echo "SUCCESS: Comparison finished, no difference is found."
+else
+  echo "FAILURE: Differences found."
+fi
+
+cd ..
+rm -rf googleapis
+
+exit $RESULT

library_generation/utilities.sh

@@ -94,12 +94,11 @@ get_gapic_opts() {
 
 remove_grpc_version() {
   find "$destination_path" -type f -name "*Grpc.java" -exec \
-  sed -i 's/value = \"by gRPC proto compiler.*/value = \"by gRPC proto compiler\",/g' {} \;
+  sed -i.bak 's/value = \"by gRPC proto compiler.*/value = \"by gRPC proto compiler\",/g' {}  \; -exec rm {}.bak \;
 }
 
 download_gapic_generator_pom_parent() {
   gapic_generator_version=$1
-  cd "$working_directory"
   if [ ! -f "gapic-generator-java-pom-parent-$gapic_generator_version.pom" ]; then
     if [[ "$gapic_generator_version" == *"-SNAPSHOT" ]]; then
       # copy a SNAPSHOT version from maven local repository.
@@ -135,14 +134,14 @@ download_tools() {
   gapic_generator_version=$1
   protobuf_version=$2
   grpc_version=$3
+  os_architecture=$4
   download_generator "$gapic_generator_version"
-  download_protobuf "$protobuf_version"
-  download_grpc_plugin "$grpc_version"
+  download_protobuf "$protobuf_version" "$os_architecture"
+  download_grpc_plugin "$grpc_version" "$os_architecture"
 }
 
 download_generator() {
   gapic_generator_version=$1
-  cd "$working_directory"
   if [ ! -f "gapic-generator-java-$gapic_generator_version.jar" ]; then
     if [[ "$gapic_generator_version" == *"-SNAPSHOT" ]]; then
       # copy a SNAPSHOT version from maven local repository.
@@ -159,32 +158,32 @@ download_generator() {
 
 download_protobuf() {
   protobuf_version=$1
-  cd "$working_directory"
+  os_architecture=$2
   if [ ! -d "protobuf-$protobuf_version.zip" ]; then
     # pull proto files and protoc from protobuf repository as maven central
     # doesn't have proto files
     download_from \
-    "https://github.com/protocolbuffers/protobuf/releases/download/v$protobuf_version/protoc-$protobuf_version-linux-x86_64.zip" \
+    "https://github.com/protocolbuffers/protobuf/releases/download/v$protobuf_version/protoc-${protobuf_version}-${os_architecture}.zip" \
     "protobuf-$protobuf_version.zip" \
     "GitHub"
     unzip -o -q "protobuf-$protobuf_version.zip" -d "protobuf-$protobuf_version"
-    cp -r "protobuf-$protobuf_version/include/google" "$working_directory"
+    cp -r "protobuf-$protobuf_version/include/google" .
     rm "protobuf-$protobuf_version.zip"
   fi
 
-  protoc_path=$working_directory/protobuf-$protobuf_version/bin
+  protoc_path=protobuf-$protobuf_version/bin
   echo "protoc version: $("$protoc_path"/protoc --version)"
 }
 
 download_grpc_plugin() {
   grpc_version=$1
-  cd "$working_directory"
-  if [ ! -f "protoc-gen-grpc-java-$grpc_version-linux-x86_64.exe" ]; then
+  os_architecture=$2
+  if [ ! -f "protoc-gen-grpc-java-$grpc_version-${os_architecture}.exe" ]; then
     # download protoc-gen-grpc-java plugin from Google maven central mirror.
     download_from \
-    "https://maven-central.storage-download.googleapis.com/maven2/io/grpc/protoc-gen-grpc-java/$grpc_version/protoc-gen-grpc-java-$grpc_version-linux-x86_64.exe" \
-    "protoc-gen-grpc-java-$grpc_version-linux-x86_64.exe"
-    chmod +x "protoc-gen-grpc-java-$grpc_version-linux-x86_64.exe"
+    "https://maven-central.storage-download.googleapis.com/maven2/io/grpc/protoc-gen-grpc-java/$grpc_version/protoc-gen-grpc-java-$grpc_version-${os_architecture}.exe" \
+    "protoc-gen-grpc-java-$grpc_version-${os_architecture}.exe"
+    chmod +x "protoc-gen-grpc-java-$grpc_version-${os_architecture}.exe"
   fi
 }