diff --git a/.github/workflows/verify_library_generation.yaml b/.github/workflows/verify_library_generation.yaml index a90f03208d..2f2b0b12b1 100644 --- a/.github/workflows/verify_library_generation.yaml +++ b/.github/workflows/verify_library_generation.yaml @@ -10,10 +10,11 @@ on: name: verify_library_generation_against_googleapis-gen jobs: verify_library_generation: - runs-on: ubuntu-22.04 strategy: matrix: java: [ 8 ] + os: [ ubuntu-22.04, macos-12 ] + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 - uses: actions/setup-java@v3 @@ -27,4 +28,5 @@ jobs: library_generation/generate_library_integration_test.sh \ -p google/bigtable/v2 \ -d google-cloud-bigtable-v2-java \ - --googleapis_gen_url https://cloud-java-bot:${{ secrets.CLOUD_JAVA_BOT_GITHUB_TOKEN }}@github.com/googleapis/googleapis-gen.git + --googleapis_gen_url https://cloud-java-bot:${{ secrets.CLOUD_JAVA_BOT_GITHUB_TOKEN }}@github.com/googleapis/googleapis-gen.git \ + --os_type ${{ matrix.os }} diff --git a/library_generation/README.md b/library_generation/README.md index e9b4b868bc..0b8db04757 100644 --- a/library_generation/README.md +++ b/library_generation/README.md @@ -108,6 +108,10 @@ The default value is `true`. Use `--include_samples` to specify the value. +### os_architecture (optional) +Choose the protoc binary type from https://github.com/protocolbuffers/protobuf/releases. +Default is "linux-x86_64". + ## An example to generate a client library ``` library_generation/generate_library.sh \ diff --git a/library_generation/gapic-generator-java-wrapper b/library_generation/gapic-generator-java-wrapper index e23d7a7df4..0f046d799e 100755 --- a/library_generation/gapic-generator-java-wrapper +++ b/library_generation/gapic-generator-java-wrapper @@ -3,5 +3,4 @@ set -e # Wrap gapic-generator-java.jar because protoc requires the plugin to be executable. -working_directory=$(dirname "$(readlink -f "$0")") -exec java -classpath "$working_directory/gapic-generator-java-$gapic_generator_version.jar" com.google.api.generator.Main +exec java -classpath "gapic-generator-java-$gapic_generator_version.jar" com.google.api.generator.Main diff --git a/library_generation/generate_library.sh b/library_generation/generate_library.sh index 134c3abc8c..8f206bc61e 100755 --- a/library_generation/generate_library.sh +++ b/library_generation/generate_library.sh @@ -41,6 +41,10 @@ case $key in include_samples="$2" shift ;; + --os_architecture) + os_architecture="$2" + shift + ;; *) echo "Invalid option: [$1]" exit 1 @@ -49,10 +53,9 @@ esac shift # past argument or value done -working_directory=$(dirname "$(readlink -f "$0")") +script_dir=$(dirname "$(readlink -f "$0")") # source utility functions -cd "$working_directory" -source ./utilities.sh +source "$script_dir"/utilities.sh if [ -z "$protobuf_version" ]; then protobuf_version=$(get_protobuf_version "$gapic_generator_version") @@ -74,25 +77,25 @@ if [ -z "$include_samples" ]; then include_samples="true" fi -cd "$working_directory" +if [ -z "$os_architecture" ]; then + os_architecture="linux-x86_64" +fi + mkdir -p "$destination_path" -destination_path="$working_directory/$destination_path" ##################### Section 0 ##################### # prepare tooling ##################################################### -cd "$working_directory" # the order of services entries in gapic_metadata.json is relevant to the # order of proto file, sort the proto files with respect to their name to # get a fixed order. proto_files=$(find "$proto_path" -type f -name "*.proto" | sort) folder_name=$(extract_folder_name "$destination_path") # download gapic-generator-java, protobuf and grpc plugin. -download_tools "$gapic_generator_version" "$protobuf_version" "$grpc_version" +download_tools "$gapic_generator_version" "$protobuf_version" "$grpc_version" "$os_architecture" ##################### Section 1 ##################### # generate grpc-*/ ##################################################### -cd "$working_directory" -"$protoc_path"/protoc "--plugin=protoc-gen-rpc-plugin=$working_directory/protoc-gen-grpc-java-$grpc_version-linux-x86_64.exe" \ +"$protoc_path"/protoc "--plugin=protoc-gen-rpc-plugin=protoc-gen-grpc-java-$grpc_version-${os_architecture}.exe" \ "--rpc-plugin_out=:$destination_path/java_grpc.jar" \ $proto_files # unzip java_grpc.jar to grpc-*/src/main/java @@ -104,9 +107,8 @@ remove_grpc_version ###################### Section 2 ##################### ## generate gapic-*/, part of proto-*/, samples/ ###################################################### -cd "$working_directory" "$protoc_path"/protoc --experimental_allow_proto3_optional \ -"--plugin=protoc-gen-java_gapic=$working_directory/gapic-generator-java-wrapper" \ +"--plugin=protoc-gen-java_gapic=$script_dir/gapic-generator-java-wrapper" \ "--java_gapic_out=metadata:$destination_path/java_gapic_srcjar_raw.srcjar.zip" \ "--java_gapic_opt=$(get_gapic_opts)" \ ${proto_files} $(search_additional_protos) @@ -125,7 +127,6 @@ if [ ! -d "$proto_dir" ]; then touch "$proto_dir"/PlaceholderFile.java fi -cd "$working_directory" # move java_gapic_srcjar/src/main to gapic-*/src. mv_src_files "gapic" "main" # remove empty files in gapic-*/src/main/java @@ -139,7 +140,6 @@ fi ##################### Section 3 ##################### # generate proto-*/ ##################################################### -cd "$working_directory" "$protoc_path"/protoc "--java_out=$destination_path/java_proto.jar" $proto_files # move java_gapic_srcjar/proto/src/main/java (generated resource name helper class) # to proto-*/src/main @@ -151,10 +151,10 @@ remove_empty_files "proto" # copy proto files to proto-*/src/main/proto for proto_src in $proto_files; do mkdir -p "$destination_path/proto-$folder_name/src/main/proto" - cp -f --parents "$proto_src" "$destination_path/proto-$folder_name/src/main/proto" + rsync -R "$proto_src" "$destination_path/proto-$folder_name/src/main/proto" done ##################### Section 4 ##################### # rm tar files ##################################################### cd "$destination_path" -rm -rf java_gapic_srcjar java_gapic_srcjar_raw.srcjar.zip java_grpc.jar java_proto.jar temp-codegen.srcjar \ No newline at end of file +rm -rf java_gapic_srcjar java_gapic_srcjar_raw.srcjar.zip java_grpc.jar java_proto.jar temp-codegen.srcjar diff --git a/library_generation/generate_library_integration_test.sh b/library_generation/generate_library_integration_test.sh index 7203b2918e..3e618d9038 100755 --- a/library_generation/generate_library_integration_test.sh +++ b/library_generation/generate_library_integration_test.sh @@ -11,6 +11,10 @@ set -xeo pipefail # `generate_library.sh`. GAPIC options to generate a library will be parsed # from proto_path/BUILD.bazel. # 4. checkout the master branch googleapis-gen repository and compare the result. + +# defaults +googleapis_gen_url="git@github.com:googleapis/googleapis-gen.git" + while [[ $# -gt 0 ]] do key="$1" @@ -27,6 +31,10 @@ case $key in googleapis_gen_url="$2" shift ;; + --os_type) + os_type="$2" + shift + ;; *) echo "Invalid option: [$1]" exit 1 @@ -43,14 +51,23 @@ get_version_from_WORKSPACE() { echo "$version" } -working_directory=$(dirname "$(readlink -f "$0")") -cd "$working_directory" +sparse_clone() { + repo_url=$1 + paths=$2 + clone_dir=$(basename "${repo_url%.*}") + rm -rf "$clone_dir" + git clone -n --depth=1 --filter=tree:0 "$repo_url" + cd "$clone_dir" + git sparse-checkout set --no-cone $paths + git checkout + cd .. +} + +script_dir=$(dirname "$(readlink -f "$0")") # checkout the master branch of googleapis/google (proto files) and WORKSPACE echo "Checking out googlapis repository..." -git clone --branch=master --depth 1 -q https://github.com/googleapis/googleapis.git -cp -r googleapis/google . -cp googleapis/WORKSPACE . -rm -rf googleapis +sparse_clone https://github.com/googleapis/googleapis.git "$proto_path WORKSPACE google/api google/rpc google/cloud/common_resources.proto" +cd googleapis # parse version of gapic-generator-java, protobuf and grpc from WORKSPACE gapic_generator_version=$(get_version_from_WORKSPACE "_gapic_generator_java_version" WORKSPACE "=") echo "The version of gapic-generator-java is $gapic_generator_version." @@ -59,7 +76,6 @@ echo "The version of protobuf is $protobuf_version" grpc_version=$(get_version_from_WORKSPACE "_grpc_version" WORKSPACE "=") echo "The version of protoc-gen-grpc-java plugin is $gapic_generator_version." # parse GAPIC options from proto_path/BUILD.bazel -cd $"$working_directory" transport="grpc" if grep -A 15 "java_gapic_library(" "$proto_path/BUILD.bazel" | grep -q "grpc+rest"; then transport="grpc+rest" @@ -73,9 +89,14 @@ if grep -A 15 "java_gapic_assembly_gradle_pkg(" "$proto_path/BUILD.bazel" | grep include_samples="true" fi echo "GAPIC options are transport=$transport, rest_numeric_enums=$rest_numeric_enums, include_samples=$include_samples." +os_architecture="linux-x86_64" +if [[ "$os_type" == *"macos"* ]]; then + os_architecture="osx-x86_64" +fi +echo "OS Architecture is $os_architecture." # generate GAPIC client library echo "Generating library from $proto_path, to $destination_path..." -"$working_directory"/generate_library.sh \ +"$script_dir"/generate_library.sh \ -p "$proto_path" \ -d "$destination_path" \ --gapic_generator_version "$gapic_generator_version" \ @@ -83,16 +104,25 @@ echo "Generating library from $proto_path, to $destination_path..." --grpc_version "$grpc_version" \ --transport "$transport" \ --rest_numeric_enums "$rest_numeric_enums" \ ---include_samples "$include_samples" +--include_samples "$include_samples" \ +--os_architecture "$os_architecture" echo "Generate library finished." echo "Checking out googleapis-gen repository..." -git clone --branch=master --depth 1 -q "$googleapis_gen_url" + +sparse_clone "$googleapis_gen_url" "$proto_path" echo "Compare generation result..." -cd "$working_directory" -diff -r "googleapis-gen/$proto_path/$destination_path" "$destination_path" -x "*gradle*" -echo "Comparison finished, no difference is found." -# clean up -cd "$working_directory" -rm -rf WORKSPACE googleapis-gen "$destination_path" +RESULT=0 +diff -r "googleapis-gen/$proto_path/$destination_path" "$destination_path" -x "*gradle*" || RESULT=$? + +if [ $RESULT == 0 ] ; then + echo "SUCCESS: Comparison finished, no difference is found." +else + echo "FAILURE: Differences found." +fi + +cd .. +rm -rf googleapis + +exit $RESULT diff --git a/library_generation/utilities.sh b/library_generation/utilities.sh index e1ef5b3e25..e77a56de39 100755 --- a/library_generation/utilities.sh +++ b/library_generation/utilities.sh @@ -94,12 +94,11 @@ get_gapic_opts() { remove_grpc_version() { find "$destination_path" -type f -name "*Grpc.java" -exec \ - sed -i 's/value = \"by gRPC proto compiler.*/value = \"by gRPC proto compiler\",/g' {} \; + sed -i.bak 's/value = \"by gRPC proto compiler.*/value = \"by gRPC proto compiler\",/g' {} \; -exec rm {}.bak \; } download_gapic_generator_pom_parent() { gapic_generator_version=$1 - cd "$working_directory" if [ ! -f "gapic-generator-java-pom-parent-$gapic_generator_version.pom" ]; then if [[ "$gapic_generator_version" == *"-SNAPSHOT" ]]; then # copy a SNAPSHOT version from maven local repository. @@ -135,14 +134,14 @@ download_tools() { gapic_generator_version=$1 protobuf_version=$2 grpc_version=$3 + os_architecture=$4 download_generator "$gapic_generator_version" - download_protobuf "$protobuf_version" - download_grpc_plugin "$grpc_version" + download_protobuf "$protobuf_version" "$os_architecture" + download_grpc_plugin "$grpc_version" "$os_architecture" } download_generator() { gapic_generator_version=$1 - cd "$working_directory" if [ ! -f "gapic-generator-java-$gapic_generator_version.jar" ]; then if [[ "$gapic_generator_version" == *"-SNAPSHOT" ]]; then # copy a SNAPSHOT version from maven local repository. @@ -159,32 +158,32 @@ download_generator() { download_protobuf() { protobuf_version=$1 - cd "$working_directory" + os_architecture=$2 if [ ! -d "protobuf-$protobuf_version.zip" ]; then # pull proto files and protoc from protobuf repository as maven central # doesn't have proto files download_from \ - "https://github.com/protocolbuffers/protobuf/releases/download/v$protobuf_version/protoc-$protobuf_version-linux-x86_64.zip" \ + "https://github.com/protocolbuffers/protobuf/releases/download/v$protobuf_version/protoc-${protobuf_version}-${os_architecture}.zip" \ "protobuf-$protobuf_version.zip" \ "GitHub" unzip -o -q "protobuf-$protobuf_version.zip" -d "protobuf-$protobuf_version" - cp -r "protobuf-$protobuf_version/include/google" "$working_directory" + cp -r "protobuf-$protobuf_version/include/google" . rm "protobuf-$protobuf_version.zip" fi - protoc_path=$working_directory/protobuf-$protobuf_version/bin + protoc_path=protobuf-$protobuf_version/bin echo "protoc version: $("$protoc_path"/protoc --version)" } download_grpc_plugin() { grpc_version=$1 - cd "$working_directory" - if [ ! -f "protoc-gen-grpc-java-$grpc_version-linux-x86_64.exe" ]; then + os_architecture=$2 + if [ ! -f "protoc-gen-grpc-java-$grpc_version-${os_architecture}.exe" ]; then # download protoc-gen-grpc-java plugin from Google maven central mirror. download_from \ - "https://maven-central.storage-download.googleapis.com/maven2/io/grpc/protoc-gen-grpc-java/$grpc_version/protoc-gen-grpc-java-$grpc_version-linux-x86_64.exe" \ - "protoc-gen-grpc-java-$grpc_version-linux-x86_64.exe" - chmod +x "protoc-gen-grpc-java-$grpc_version-linux-x86_64.exe" + "https://maven-central.storage-download.googleapis.com/maven2/io/grpc/protoc-gen-grpc-java/$grpc_version/protoc-gen-grpc-java-$grpc_version-${os_architecture}.exe" \ + "protoc-gen-grpc-java-$grpc_version-${os_architecture}.exe" + chmod +x "protoc-gen-grpc-java-$grpc_version-${os_architecture}.exe" fi }