Merge branch 'master' into license

adoptium · Mar 30, 2024 · 6294648 · 6294648
2 parents 1bec3be + 494f914
commit 6294648
Show file tree

Hide file tree

Showing 6 changed files with 135 additions and 57 deletions.
diff --git a/README.md b/README.md
@@ -233,6 +233,11 @@ For reference, OpenJDK version numbers look like 1.8.0_162-b12 (for Java 8) or
 (162) or the 3rd position in the semVer version string (4).
 This is typically used in conjunction with -b.
 
+--use-adoptium-devkit <Adoptium DevKit release tag>
+Download and use the given DevKit from https://github.com/adoptium/devkit-binaries/releases.
+The DevKit is downloaded and unpacked to WORKSPACE_DIR/WORKING_DIR/devkit
+and will add the configure arg --with-devkit=WORKSPACE_DIR/WORKING_DIR/devkit.
+
 --use-jep319-certs
 Use certs defined in JEP319 in Java 8/9. Deprecated, has no effect.
 

diff --git a/build-farm/platform-specific-configurations/linux.sh b/build-farm/platform-specific-configurations/linux.sh
@@ -332,6 +332,8 @@ fi
 
 if [[ "${CONFIGURE_ARGS}" =~ .*"--with-devkit=".* ]]; then
   echo "Using gcc from DevKit toolchain specified in configure args"
+elif [[ "${BUILD_ARGS}" =~ .*"--use-adoptium-devkit".* ]]; then
+  echo "Using gcc from Adoptium DevKit toolchain specified in --use-adoptium-devkit build args"
 else 
   if [ "${VARIANT}" == "${BUILD_VARIANT_DRAGONWELL}" ] && [ "$JAVA_FEATURE_VERSION" -eq 11 ] && [ -r /usr/local/gcc9/ ] && [ "${ARCHITECTURE}" == "aarch64" ]; then
     # GCC9 rather than 10 requested by Alibaba for now

diff --git a/sbin/build.sh b/sbin/build.sh
@@ -73,6 +73,13 @@ addConfigureArgIfValueIsNotEmpty() {
   fi
 }
 
+# Configure the DevKit if required
+configureDevKitConfigureParameter() {
+  if [[ -n "${BUILD_CONFIG[USE_ADOPTIUM_DEVKIT]}" ]]; then
+    addConfigureArg "--with-devkit=" "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[WORKING_DIR]}/devkit"
+  fi
+} 
+
 # Configure the boot JDK
 configureBootJDKConfigureParameter() {
   addConfigureArgIfValueIsNotEmpty "--with-boot-jdk=" "${BUILD_CONFIG[JDK_BOOT_DIR]}"
@@ -534,6 +541,7 @@ configureZlibLocation() {
 configureCommandParameters() {
   configureVersionStringParameter
   configureBootJDKConfigureParameter
+  configureDevKitConfigureParameter
   configureShenandoahBuildParameter
   configureMacOSCodesignParameter
   configureDebugParameters

diff --git a/sbin/common/config_init.sh b/sbin/common/config_init.sh
@@ -96,6 +96,7 @@ TARGET_DIR
 TARGET_FILE_NAME
 TMP_CONTAINER_NAME
 TMP_SPACE_BUILD
+USE_ADOPTIUM_DEVKIT
 USE_DOCKER
 USE_JEP319_CERTS
 USE_SSH
@@ -119,6 +120,7 @@ WORKSPACE_DIR
 #  <WORKSPACE_DIR>/config                              Configuration                        /openjdk/config             $(pwd)/workspace/config
 #  <WORKSPACE_DIR>/<WORKING_DIR>                       Build area                           /openjdk/build              $(pwd)/workspace/build/
 #  <WORKSPACE_DIR>/<WORKING_DIR>/<OPENJDK_SOURCE_DIR>  Source code                          /openjdk/build/src          $(pwd)/workspace/build/src
+#  <WORKSPACE_DIR>/<WORKING_DIR>/devkit                DevKit download                      /openjdk/build/devkit       $(pwd)/workspace/build/devkit
 #  <WORKSPACE_DIR>/target                              Destination of built artifacts       /openjdk/target             $(pwd)/workspace/target
 
 # Helper code to perform index lookups by name
@@ -353,6 +355,9 @@ function parseConfigurationArguments() {
         "--use-jep319-certs" )
         BUILD_CONFIG[USE_JEP319_CERTS]=true;;
 
+        "--use-adoptium-devkit")
+        BUILD_CONFIG[USE_ADOPTIUM_DEVKIT]="$1"; shift;;
+
         "--user-openjdk-build-root-directory" )
         BUILD_CONFIG[USER_OPENJDK_BUILD_ROOT_DIRECTORY]="$1"; shift;;
 
@@ -597,6 +602,9 @@ function configDefaults() {
 
   BUILD_CONFIG[CLEAN_LIBS]=false
 
+  # Default to no Adoptium DevKit
+  BUILD_CONFIG[USE_ADOPTIUM_DEVKIT]=""
+
   # By default dont backport JEP318 certs to < Java 10
   BUILD_CONFIG[USE_JEP319_CERTS]=false
 

diff --git a/sbin/prepareWorkspace.sh b/sbin/prepareWorkspace.sh
@@ -328,21 +328,8 @@ checkingAndDownloadingAlsa() {
     curl -o "alsa-lib.tar.bz2" "$ALSA_BUILD_URL"
     curl -o "alsa-lib.tar.bz2.sig" "https://www.alsa-project.org/files/pub/lib/alsa-lib-${ALSA_LIB_VERSION}.tar.bz2.sig"
 
-    ## This affects riscv64 & Alpine docker images and also evaluation pipelines
-    if ( [ -r /etc/alpine-release ] && [ "$(pwd | wc -c)" -gt 83 ] ) || \
-       ( [ "${BUILD_CONFIG[OS_KERNEL_NAME]}" == "linux" ] && [ "${BUILD_CONFIG[OS_ARCHITECTURE]}" == "riscv64" ] && [ "$(pwd | wc -c)" -gt 83 ] ); then
-        # Use /tmp in preference to $HOME as fails gpg operation if PWD > 83 characters
-        # Also cannot create ~/.gpg-temp within a docker context
-        GNUPGHOME="$(mktemp -d /tmp/.gpg-temp.XXXXXX)"
-    else
-        GNUPGHOME="${BUILD_CONFIG[WORKSPACE_DIR]:-$PWD}/.gpg-temp"
-    fi
-    if [ ! -d "$GNUPGHOME" ]; then
-        mkdir -m 700 "$GNUPGHOME"
-    fi
-    export GNUPGHOME
+    setupGpg
 
-    echo "GNUPGHOME=$GNUPGHOME"
     # Should we clear this directory up after checking?
     # Would this risk removing anyone's existing dir with that name?
     # Erring on the side of caution for now
@@ -575,6 +562,67 @@ prepareMozillaCacerts() {
     fi
 }
 
+# Create and setup GNUPGHOME
+setupGpg() {
+    ## This affects riscv64 & Alpine docker images and also evaluation pipelines
+    if ( [ -r /etc/alpine-release ] && [ "$(pwd | wc -c)" -gt 83 ] ) || \
+       ( [ "${BUILD_CONFIG[OS_KERNEL_NAME]}" == "linux" ] && [ "${BUILD_CONFIG[OS_ARCHITECTURE]}" == "riscv64" ] && [ "$(pwd | wc -c)" -gt 83 ] ); then
+        # Use /tmp in preference to $HOME as fails gpg operation if PWD > 83 characters
+        # Also cannot create ~/.gpg-temp within a docker context
+        GNUPGHOME="$(mktemp -d /tmp/.gpg-temp.XXXXXX)"
+    else
+        GNUPGHOME="${BUILD_CONFIG[WORKSPACE_DIR]:-$PWD}/.gpg-temp"
+    fi
+    if [ ! -d "$GNUPGHOME" ]; then
+        mkdir -m 700 "$GNUPGHOME"
+    fi
+    export GNUPGHOME
+
+    echo "GNUPGHOME=$GNUPGHOME"
+}
+
+# Download the required DevKit if necessary
+downloadDevkit() {
+  if [[ -n "${BUILD_CONFIG[USE_ADOPTIUM_DEVKIT]}" ]]; then
+    rm -rf "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[WORKING_DIR]}/devkit"
+    mkdir -p "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[WORKING_DIR]}/devkit"
+
+    local devkit_tar="${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[WORKING_DIR]}/devkit/devkit.tar.xz"
+
+    setupGpg
+
+    # Determine DevKit tarball to download for this arch and release
+    local devkitUrl="https://github.com/adoptium/devkit-binaries/releases/download/${BUILD_CONFIG[USE_ADOPTIUM_DEVKIT]}"
+    local devkit_target="${BUILD_CONFIG[OS_ARCHITECTURE]}-linux-gnu"
+    local devkit="devkit-${BUILD_CONFIG[USE_ADOPTIUM_DEVKIT]}-${devkit_target}"
+
+    # Download tarball and GPG sig
+    echo "Downloading DevKit : ${devkitUrl}/${devkit}.tar.xz"
+    curl -L --fail --silent --show-error -o "${devkit_tar}" "${devkitUrl}/${devkit}.tar.xz"
+    curl -L --fail --silent --show-error -o "${devkit_tar}.sig" "${devkitUrl}/${devkit}.tar.xz.sig"
+
+    # GPG verify
+    gpg --keyserver keyserver.ubuntu.com --recv-keys 3B04D753C9050D9A5D343F39843C48A565F8F04B
+    echo -e "5\ny\n" |  gpg --batch --command-fd 0 --expert --edit-key 3B04D753C9050D9A5D343F39843C48A565F8F04B trust;
+    gpg --verify "${devkit_tar}.sig" "${devkit_tar}" || exit 1
+
+    tar xpJf "${devkit_tar}" -C "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[WORKING_DIR]}/devkit"
+    rm "${devkit_tar}"
+    rm "${devkit_tar}.sig"
+
+    # Validate devkit.info matches value passed in and current architecture
+    local devkitInfo="${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[WORKING_DIR]}/devkit/devkit.info"
+    if ! grep "ADOPTIUM_DEVKIT_RELEASE=${BUILD_CONFIG[USE_ADOPTIUM_DEVKIT]}" "${devkitInfo}" || ! grep "ADOPTIUM_DEVKIT_TARGET=${devkit_target}" "${devkitInfo}"; then
+        echo "ERROR: Devkit does not match required release and architecture:"
+        echo "       Required:   ADOPTIUM_DEVKIT_RELEASE=${BUILD_CONFIG[USE_ADOPTIUM_DEVKIT]}"
+        echo "       Downloaded: $(grep ADOPTIUM_DEVKIT_RELEASE= "${devkitInfo}")"
+        echo "       Required:   ADOPTIUM_DEVKIT_TARGET=${devkit_target}"
+        echo "       Downloaded: $(grep ADOPTIUM_DEVKIT_TARGET= "${devkitInfo}")"
+        exit 1
+    fi
+  fi
+}
+
 # Download all of the dependencies for OpenJDK (Alsa, FreeType etc.)
 downloadingRequiredDependencies() {
   if [[ "${BUILD_CONFIG[CLEAN_LIBS]}" == "true" ]]; then
@@ -693,6 +741,7 @@ function configureWorkspace() {
   if [[ "${BUILD_CONFIG[ASSEMBLE_EXPLODED_IMAGE]}" != "true" ]]; then
     createWorkspace
     downloadingRequiredDependencies
+    downloadDevkit
     relocateToTmpIfNeeded
     checkoutAndCloneOpenJDKGitRepo
     applyPatches

diff --git a/tooling/reproducible/windows_repro_build_compare.sh b/tooling/reproducible/windows_repro_build_compare.sh
@@ -39,7 +39,8 @@ TARBALL_URL="$2"
 # These Values Should Be Updated To Reflect The Build Environment
 # The Defaults Below Are Suitable For An Adoptium Windows Build Environment
 # Which Has Been Created Via The Ansible Infrastructure Playbooks
-WORK_DIR="/cmp$(date +%Y%m%d%H%M%S)"
+
+WORK_DIR="/cygdrive/c/comp-jdk-build"
 ANT_VERSION="1.10.5"
 ANT_CONTRIB_VERSION="1.0b3"
 ANT_BASE_PATH="/cygdrive/c/apache-ant"
@@ -55,12 +56,13 @@ SIGNTOOL_BASE="C:/Program Files (x86)/Windows Kits/10"
 
 # Define What Are Configure Args & Redundant Args
 # This MAY Need Updating If Additional Configure Args Are Passed
-CONFIG_ARGS=("--disable-warnings-as-errors" "--disable-ccache" "--with-toolchain-version" "--with-ucrt-dll-dir")
+CONFIG_ARGS=("--disable-warnings-as-errors" "--disable-ccache" "--with-toolchain-version" "--with-ucrt-dll-dir" "--with-version-opt")
 NOTUSE_ARGS=("--assemble-exploded-image" "--configure-args")
 
 # Addiitonal Working Variables Defined For Use By This Script
 SBOMLocalPath="$WORK_DIR/src_sbom.json"
 DISTLocalPath="$WORK_DIR/src_jdk_dist.zip"
+ScriptPath="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
 # Function to check if a string is a valid URL
 is_url() {
@@ -92,7 +94,7 @@ Create_WorkDir() {
     echo "Folder Exists - Removing '$WORK_DIR'"
   fi
   echo "Creating $WORK_DIR"
-  mkdir "$WORK_DIR"
+  mkdir -p "$WORK_DIR"
 }
 
 # Function To Check The SBOM
@@ -138,37 +140,41 @@ Check_Parameters() {
 }
 
 Install_PreReqs() {
-  # Install Cygwin Package Manager If Not Present
-  echo "Checking If Apt-Cyg Is Already Installed"
-  if [ -f /usr/local/bin/apt-cyg ]; then
-    echo "Skipping apt-cyg Install"
-    APTCYG_INSTALLED="True"
-  else
-    echo "Installing apt-cyg"
-    APTCYG_INSTALLED="False"
-    wget -q -O "./apt-cyg" "https://raw.githubusercontent.com/transcode-open/apt-cyg/master/apt-cyg"
-    ACTSHASUM=$(sha256sum "apt-cyg" | awk '{print $1}')
-    EXPSHASUM="d020050e2cb56fec990f16fd10695e153afd064cb0839ba935247b5a9e4c29a0"
-
-    if [ "$ACTSHASUM" == "$EXPSHASUM" ]; then
-      chmod +x apt-cyg
-      mv apt-cyg /usr/local/bin
-    else
-      echo "Checksum Is Not OK - Exiting"
-      exit 1
-    fi
+  # Check For JQ & Install Apt-Cyg & JQ Where Not Available
+  if ! command -v jq &> /dev/null; then
+      echo "WARNING: JQ is not installed. Attempting To Install Via Apt-Cyg"
+      echo "Checking If Apt-Cyg Is Already Installed"
+      if [ -f /usr/local/bin/apt-cyg ]; then
+        echo "Skipping apt-cyg Install"
+        APTCYG_INSTALLED="True"
+      else
+        echo "Installing apt-cyg"
+        APTCYG_INSTALLED="False"
+        wget -q -O "./apt-cyg" "https://raw.githubusercontent.com/transcode-open/apt-cyg/master/apt-cyg"
+        ACTSHASUM=$(sha256sum "apt-cyg" | awk '{print $1}')
+        EXPSHASUM="d020050e2cb56fec990f16fd10695e153afd064cb0839ba935247b5a9e4c29a0"
+        if [ "$ACTSHASUM" == "$EXPSHASUM" ]; then
+          chmod +x apt-cyg
+          mv apt-cyg /usr/local/bin
+        else
+          echo "Checksum Is Not OK - Exiting"
+          exit 1
+        fi
+      fi
+
+      echo "Checking If JQ Is Already Installed"
+      if [ -f /usr/local/bin/jq ]; then
+        echo "Skipping JQ Install"
+        APTJQ_INSTALLED="True"
+      else
+        echo "Installing JQ via APTCYG"
+        APTJQ_INSTALLED="False"
+        apt-cyg install jq libjq1 libonig5
+      fi
   fi
 
   # Install JQ Where Not Already Installed
-  echo "Checking If JQ Is Already Installed"
-  if [ -f /usr/local/bin/jq ]; then
-    echo "Skipping JQ Install"
-    APTJQ_INSTALLED="True"
-  else
-    echo "Installing JQ via APTCYG"
-    APTJQ_INSTALLED="False"
-    apt-cyg install jq libjq1 libonig5
-  fi
+
 }
 
 Get_SBOM_Values() {
@@ -281,7 +287,7 @@ Get_SBOM_Values() {
       echo "Temurin Build Arguments: $buildArgs"
       export buildArgs
   else
-      echo "ERROR: Temurin Build Version not found in the SBOM."
+      echo "ERROR: Temurin Build Arguments not found in the SBOM."
       echo "This Is A Mandatory Element"
       exit 1
   fi
@@ -567,7 +573,7 @@ Prepare_Env_For_Build() {
   # Loop through the words
   for word in "${words[@]}"; do
     # Check if the word starts with '--'
-    if [[ $word == --* ]]; then
+    if [[ $word == --* ]] || [[ $word == -b* ]]; then
       # If a parameter already exists, store it in the params array
       if [[ -n $param ]]; then
         params+=("$param=$value")
@@ -580,7 +586,7 @@ Prepare_Env_For_Build() {
     fi
   done
 
-  # Add the last parameter to the array
+    # Add the last parameter to the array
   params+=("$param = $value")
 
   # Read the separated parameters and values into a new array
@@ -600,13 +606,15 @@ Prepare_Env_For_Build() {
     fixed_value=$(echo "$prepped_value" | awk '{$1=$1};1')
 
     # Handle Special parameters
+    if [ "$fixed_param" == "-b" ]; then fixed_value="$fixed_value " ; fi
     if [ "$fixed_param" == "--jdk-boot-dir" ]; then fixed_value="$BOOTJDK_HOME " ; fi
     if [ "$fixed_param" == "--freetype-dir" ]; then fixed_value="$fixed_value " ; fi
     if [ "$fixed_param" == "--with-toolchain-version" ]; then fixed_value="$visualStudioVersion " ; fi
-    if [ "$fixed_param" == "--with-ucrt-dll-dir" ]; then fixed_value="temporary_speech_mark_placeholder${UCRT_PARAM_PATH}temporary_speech_mark_placeholder" ; fi
+    if [ "$fixed_param" == "--with-ucrt-dll-dir" ]; then fixed_value="temporary_speech_mark_placeholder${UCRT_PARAM_PATH}temporary_speech_mark_placeholder " ; fi
     if [ "$fixed_param" == "--target-file-name" ]; then target_file="$fixed_value" ; fixed_value="$fixed_value " ; fi
     if [ "$fixed_param" == "--tag" ]; then fixed_value="$fixed_value " ; fi
 
+
     # Fix Build Variant Parameter To Strip JDK Version
 
     if [ "$fixed_param" == "--build-variant" ] ; then
@@ -628,7 +636,7 @@ Prepare_Env_For_Build() {
       # Add Config Arg To New Array
 
       # Handle Windows Param Names In Config Args (Replace Space with =)
-      if [ "$fixed_param" == "--with-toolchain-version" ] || [ "$fixed_param" == "--with-ucrt-dll-dir" ] ; then
+      if [ "$fixed_param" == "--with-toolchain-version" ] || [ "$fixed_param" == "--with-ucrt-dll-dir" ] ||  [ "$fixed_param" == "--with-version-opt" ] ; then
         STRINGTOADD="$fixed_param=$fixed_value"
         CONFIG_ARRAY+=("$STRINGTOADD")
       else
@@ -675,7 +683,6 @@ Build_JDK() {
 Compare_JDK() {
   echo "Comparing JDKs"
   echo ""
-  cd "$WORK_DIR"
   mkdir "$WORK_DIR/compare"
   cp "$WORK_DIR/src_jdk_dist.zip" "$WORK_DIR/compare"
   cp "$WORK_DIR/built_jdk.zip" "$WORK_DIR/compare"
@@ -749,15 +756,12 @@ Compare_JDK() {
   export JAVA_HOME=$BOOTJDK_HOME
   export PATH=$JAVA_HOME/bin:$PATH
 
-  # Run Comparison Script
-  cd "$WORK_DIR/compare"
-
   CPW=$(cygpath -u "$SIGNPATHWIN")
   export PATH="$PATH:$CPW"
 
   # Run Comparison Script
-  cd "$WORK_DIR/compare"
-  ./repro_compare.sh temurin src_jdk temurin tar_jdk CYGWIN
+  echo "cd $WORK_DIR/compare && ./repro_compare.sh temurin src_jdk temurin tar_jdk CYGWIN 2>&1" | sh &
+  wait
 
   # Display The Content Of repro_diff.out
   echo ""
@@ -767,7 +771,9 @@ Compare_JDK() {
   cat "$WORK_DIR/compare/repro_diff.out"
   echo ""
   echo "---------------------------------------------"
-  mv "$WORK_DIR/compare/repro_diff.out" "$WORK_DIR"
+  echo "Copying Output To $(dirname "$0")"
+  cp "$WORK_DIR/compare/repro_diff.out" "$ScriptPath"
+
 }
 
 Clean_Up_Everything() {