From 629efb09bbc8b45288c14e8d32c2042e670a41ab Mon Sep 17 00:00:00 2001
From: pxLi <pxli@nyu.edu>
Date: Tue, 29 Sep 2020 21:32:54 +0800
Subject: [PATCH] Migrate nightly&integration pipelines to blossom [skip ci]
 (#862)

* migrate nightly to blossom

Signed-off-by: Peixin Li <pxli@nyu.edu>

* fix license 2019 and 2020 for dockerfiles

* fix ARGs in dockerfiles of integration test
---
 .../Dockerfile-blossom.integration.centos7    | 56 +++++++++++++++++++
 jenkins/Dockerfile.integration.centos7        | 12 ++--
 jenkins/Dockerfile.integration.ubuntu16       |  2 +-
 jenkins/spark-tests.sh                        |  9 ++-
 4 files changed, 70 insertions(+), 9 deletions(-)
 create mode 100644 jenkins/Dockerfile-blossom.integration.centos7

diff --git a/jenkins/Dockerfile-blossom.integration.centos7 b/jenkins/Dockerfile-blossom.integration.centos7
new file mode 100644
index 00000000000..23ce6f4bd59
--- /dev/null
+++ b/jenkins/Dockerfile-blossom.integration.centos7
@@ -0,0 +1,56 @@
+#
+# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+###
+#
+# Arguments:
+#    CUDA_VER=10.1 or 10.2
+#    CUDF_VER=0.15 or 0.16-SNAPSHOT
+#    URM_URL=<maven repo url>
+###
+ARG CUDA_VER=10.1
+ARG CUDF_VER
+ARG URM_URL
+
+FROM nvidia/cuda:${CUDA_VER}-runtime-centos7
+
+#Install java-8, maven, docker image
+RUN yum update -y && \
+    yum install -y centos-release-scl && \
+    yum install -y java-1.8.0-openjdk-devel wget expect
+
+# The default mvn verision is 3.05 on centos7 docker container.
+# The plugin: net.alchim31.maven requires a higher mvn version.
+ENV MAVEN_HOME "/usr/local/apache-maven-3.6.3"
+RUN wget ${URM_URL}/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.tar.gz -P /usr/local && \
+    tar xzvf $MAVEN_HOME-bin.tar.gz -C /usr/local && \
+    rm -f $MAVEN_HOME-bin.tar.gz
+
+RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh && \
+    /bin/bash ~/miniconda.sh -b -p /opt/conda && \
+    rm -f ~/miniconda.sh
+
+ENV PATH="/opt/conda/bin:$MAVEN_HOME/bin:${PATH}"
+
+# 'pyarrow' and 'pandas' will be installed as the dependencies of cudf below
+RUN conda install -y -c rapidsai -c rapidsai-nightly -c nvidia -c conda-forge -c defaults cudf=${CUDF_VER} python=3.7 cudatoolkit=${CUDA_VER} && \
+    conda install -y spacy && python -m spacy download en_core_web_sm && \
+    conda install -y -c anaconda pytest requests && \
+    conda install -y -c conda-forge sre_yield && \
+    conda clean -ay
+
+# Set ENV for mvn
+ENV JAVA_HOME "/usr/lib/jvm/java-1.8.0-openjdk"
diff --git a/jenkins/Dockerfile.integration.centos7 b/jenkins/Dockerfile.integration.centos7
index dcb3579c9dd..ec6066dab37 100644
--- a/jenkins/Dockerfile.integration.centos7
+++ b/jenkins/Dockerfile.integration.centos7
@@ -16,17 +16,17 @@
 
 ###
 #
-# Arguments: CUDA_VER=10.1 or 10.2
-#
+# Arguments:
+#    CUDA_VER=10.1 or 10.2
+#    CUDF_VER=0.15 or 0.16-SNAPSHOT
+#    URM_URL=<maven repo url>
 ###
-ARG CUDA_VER=10.1
-
-FROM nvidia/cuda:${CUDA_VER}-runtime-centos7
-
 ARG CUDA_VER=10.1
 ARG CUDF_VER
 ARG URM_URL
 
+FROM nvidia/cuda:${CUDA_VER}-runtime-centos7
+
 #Install java-8, maven, docker image
 RUN yum update -y && \
     yum install -y centos-release-scl && \
diff --git a/jenkins/Dockerfile.integration.ubuntu16 b/jenkins/Dockerfile.integration.ubuntu16
index 75d0d28b766..a608863c43e 100644
--- a/jenkins/Dockerfile.integration.ubuntu16
+++ b/jenkins/Dockerfile.integration.ubuntu16
@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2019-2020, NVIDIA CORPORATION. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/jenkins/spark-tests.sh b/jenkins/spark-tests.sh
index bca50bbee6a..8bc6549907b 100755
--- a/jenkins/spark-tests.sh
+++ b/jenkins/spark-tests.sh
@@ -17,6 +17,8 @@
 
 set -ex
 
+nvidia-smi
+
 . jenkins/version-def.sh
 
 ARTF_ROOT="$WORKSPACE/jars"
@@ -59,7 +61,9 @@ tar zxf $SPARK_HOME.tgz -C $ARTF_ROOT && \
 PARQUET_PERF="$WORKSPACE/integration_tests/src/test/resources/parquet_perf"
 PARQUET_ACQ="$WORKSPACE/integration_tests/src/test/resources/parquet_acq"
 OUTPUT="$WORKSPACE/output"
-BASE_SPARK_SUBMIT_ARGS="--master spark://$HOSTNAME:7077 --executor-memory 32G \
+BASE_SPARK_SUBMIT_ARGS="--master spark://$HOSTNAME:7077 \
+    --executor-memory 12G \
+    --total-executor-cores 6 \
     --conf spark.sql.shuffle.partitions=12 \
     --conf spark.driver.extraClassPath=${CUDF_JAR}:${RAPIDS_PLUGIN_JAR} \
     --conf spark.executor.extraClassPath=${CUDF_JAR}:${RAPIDS_PLUGIN_JAR} \
@@ -75,7 +79,8 @@ CUDF_UDF_TEST_ARGS="--conf spark.rapids.memory.gpu.allocFraction=0.1 \
     --conf spark.rapids.python.memory.gpu.allocFraction=0.1 \
     --conf spark.rapids.python.concurrentPythonWorkers=2 \
     --conf spark.executorEnv.PYTHONPATH=${RAPIDS_FILE_NAME} \
-    --py-files ${RAPIDS_PLUGIN_JAR}"
+    --conf spark.pyspark.python=/opt/conda/bin/python \
+    --py-files ${RAPIDS_PLUGIN_JAR}" # explicitly specify python binary path in env w/ multiple python versions
 
 TEST_PARAMS="$SPARK_VER $PARQUET_PERF $PARQUET_ACQ $OUTPUT"