-
Notifications
You must be signed in to change notification settings - Fork 527
/
build.sh
executable file
·247 lines (188 loc) · 9.04 KB
/
build.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
#!/bin/bash
# Copyright (c) 2018-2021, NVIDIA CORPORATION.
##############################################
# cuML GPU build and test script for CI #
##############################################
set -e
NUMARGS=$#
ARGS=$*
# Arg parsing function
function hasArg {
(( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ")
}
# Set path and build parallel level
export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH
export PARALLEL_LEVEL=${PARALLEL_LEVEL:-4}
# Set home to the job's workspace
export HOME=$WORKSPACE
# Determine CUDA release version
export CUDA_REL=${CUDA_VERSION%.*}
# Switch to project root; also root of repo checkout
cd $WORKSPACE
# Parse git describe
export GIT_DESCRIBE_TAG=`git describe --tags`
export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'`
################################################################################
# SETUP - Check environment
################################################################################
gpuci_logger "Check environment"
env
gpuci_logger "Check GPU usage"
nvidia-smi
gpuci_logger "Activate conda env"
. /opt/conda/etc/profile.d/conda.sh
conda activate rapids
gpuci_logger "Install dependencies"
gpuci_conda_retry install -c conda-forge -c rapidsai -c rapidsai-nightly -c nvidia \
"cudatoolkit=${CUDA_REL}" \
"cudf=${MINOR_VERSION}" \
"rmm=${MINOR_VERSION}" \
"libcumlprims=${MINOR_VERSION}" \
"dask-cudf=${MINOR_VERSION}" \
"dask-cuda=${MINOR_VERSION}" \
"ucx-py=${MINOR_VERSION}" \
"xgboost=1.3.0dev.rapidsai${MINOR_VERSION}" \
"rapids-build-env=${MINOR_VERSION}.*" \
"rapids-notebook-env=${MINOR_VERSION}.*" \
"rapids-doc-env=${MINOR_VERSION}.*"
# https://docs.rapids.ai/maintainers/depmgmt/
# gpuci_conda_retry remove --force rapids-build-env rapids-notebook-env
# gpuci_conda_retry install -y "your-pkg=1.0.0"
gpuci_logger "Install contextvars if needed"
py_ver=$(python -c "import sys; print('.'.join(map(str, sys.version_info[:2])))")
if [ "$py_ver" == "3.6" ];then
conda install contextvars
fi
gpuci_logger "Install the master version of dask and distributed"
set -x
pip install "git+https://github.com/dask/distributed.git@master" --upgrade --no-deps
pip install "git+https://github.com/dask/dask.git@master" --upgrade --no-deps
set +x
gpuci_logger "Check compiler versions"
python --version
$CC --version
$CXX --version
gpuci_logger "Check conda environment"
conda info
conda config --show-sources
conda list --show-channel-urls
gpuci_logger "Adding ${CONDA_PREFIX}/lib to LD_LIBRARY_PATH"
export LD_LIBRARY_PATH_CACHED=$LD_LIBRARY_PATH
export LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH
if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
gpuci_logger "Building doxygen C++ docs"
$WORKSPACE/build.sh cppdocs -v
################################################################################
# BUILD - Build libcuml, cuML, and prims from source
################################################################################
gpuci_logger "Build from source"
$WORKSPACE/build.sh clean libcuml cuml prims bench -v --codecov
gpuci_logger "Resetting LD_LIBRARY_PATH"
cd $WORKSPACE
################################################################################
# TEST - Run GoogleTest and py.tests for libcuml and cuML
################################################################################
if hasArg --skip-tests; then
gpuci_logger "Skipping Tests"
exit 0
fi
gpuci_logger "Check GPU usage"
nvidia-smi
gpuci_logger "GoogleTest for libcuml"
set -x
cd $WORKSPACE/cpp/build
GTEST_OUTPUT="xml:${WORKSPACE}/test-results/libcuml_cpp/" ./test/ml
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH_CACHED
export LD_LIBRARY_PATH_CACHED=""
gpuci_logger "Python pytest for cuml"
cd $WORKSPACE/python
pytest --cache-clear --basetemp=${WORKSPACE}/cuml-cuda-tmp --junitxml=${WORKSPACE}/junit-cuml.xml -v -s -m "not memleak" --durations=50 --timeout=300 --ignore=cuml/test/dask --ignore=cuml/raft --cov-config=.coveragerc --cov=cuml --cov-report=xml:${WORKSPACE}/python/cuml/cuml-coverage.xml --cov-report term
timeout 7200 sh -c "pytest cuml/test/dask --cache-clear --basetemp=${WORKSPACE}/cuml-mg-cuda-tmp --junitxml=${WORKSPACE}/junit-cuml-mg.xml -v -s -m 'not memleak' --durations=50 --timeout=300"
################################################################################
# TEST - Run notebook tests
################################################################################
set +e -Eo pipefail
EXITCODE=0
trap "EXITCODE=1" ERR
gpuci_logger "Notebook tests"
${WORKSPACE}/ci/gpu/test-notebooks.sh 2>&1 | tee nbtest.log
python ${WORKSPACE}/ci/utils/nbtestlog2junitxml.py nbtest.log
################################################################################
# TEST - Run GoogleTest for ml-prims
################################################################################
gpuci_logger "Run ml-prims test"
cd $WORKSPACE/cpp/build
GTEST_OUTPUT="xml:${WORKSPACE}/test-results/prims/" ./test/prims
################################################################################
# TEST - Run GoogleTest for ml-prims, but with cuda-memcheck enabled
################################################################################
if [ "$BUILD_MODE" = "branch" ] && [ "$BUILD_TYPE" = "gpu" ]; then
gpuci_logger "GoogleTest for ml-prims with cuda-memcheck enabled..."
cd $WORKSPACE/cpp/build
python ../scripts/cuda-memcheck.py -tool memcheck -exe ./test/prims
fi
else
#Project Flash
export LIBCUML_BUILD_DIR="$WORKSPACE/ci/artifacts/cuml/cpu/conda_work/cpp/build"
export LD_LIBRARY_PATH="$LIBCUML_BUILD_DIR:$LD_LIBRARY_PATH"
if hasArg --skip-tests; then
gpuci_logger "Skipping Tests"
exit 0
fi
gpuci_logger "Check GPU usage"
nvidia-smi
gpuci_logger "Update binaries"
cd $LIBCUML_BUILD_DIR
chrpath -d libcuml.so
chrpath -d libcuml++.so
patchelf --replace-needed `patchelf --print-needed libcuml++.so | grep faiss` libfaiss.so libcuml++.so
gpuci_logger "GoogleTest for libcuml"
cd $LIBCUML_BUILD_DIR
chrpath -d ./test/ml
patchelf --replace-needed `patchelf --print-needed ./test/ml | grep faiss` libfaiss.so ./test/ml
GTEST_OUTPUT="xml:${WORKSPACE}/test-results/libcuml_cpp/" ./test/ml
CONDA_FILE=`find $WORKSPACE/ci/artifacts/cuml/cpu/conda-bld/ -name "libcuml*.tar.bz2"`
CONDA_FILE=`basename "$CONDA_FILE" .tar.bz2` #get filename without extension
CONDA_FILE=${CONDA_FILE//-/=} #convert to conda install
gpuci_logger "Installing $CONDA_FILE"
conda install -c $WORKSPACE/ci/artifacts/cuml/cpu/conda-bld/ "$CONDA_FILE"
gpuci_logger "Building cuml"
"$WORKSPACE/build.sh" -v cuml --codecov
gpuci_logger "Python pytest for cuml"
cd $WORKSPACE/python
pytest --cache-clear --basetemp=${WORKSPACE}/cuml-cuda-tmp --junitxml=${WORKSPACE}/junit-cuml.xml -v -s -m "not memleak" --durations=50 --timeout=300 --ignore=cuml/test/dask --ignore=cuml/raft --cov-config=.coveragerc --cov=cuml --cov-report=xml:${WORKSPACE}/python/cuml/cuml-coverage.xml --cov-report term
timeout 7200 sh -c "pytest cuml/test/dask --cache-clear --basetemp=${WORKSPACE}/cuml-mg-cuda-tmp --junitxml=${WORKSPACE}/junit-cuml-mg.xml -v -s -m 'not memleak' --durations=50 --timeout=300"
################################################################################
# TEST - Run notebook tests
################################################################################
gpuci_logger "Notebook tests"
set +e -Eo pipefail
EXITCODE=0
trap "EXITCODE=1" ERR
${WORKSPACE}/ci/gpu/test-notebooks.sh 2>&1 | tee nbtest.log
python ${WORKSPACE}/ci/utils/nbtestlog2junitxml.py nbtest.log
################################################################################
# TEST - Run GoogleTest for ml-prims
################################################################################
gpuci_logger "Run ml-prims test"
cd $LIBCUML_BUILD_DIR
chrpath -d ./test/prims
patchelf --replace-needed `patchelf --print-needed ./test/prims | grep faiss` libfaiss.so ./test/prims
GTEST_OUTPUT="xml:${WORKSPACE}/test-results/prims/" ./test/prims
################################################################################
# TEST - Run GoogleTest for ml-prims, but with cuda-memcheck enabled
################################################################################
if [ "$BUILD_MODE" = "branch" ] && [ "$BUILD_TYPE" = "gpu" ]; then
logger "GoogleTest for ml-prims with cuda-memcheck enabled..."
cd $WORKSPACE/ci/artifacts/cuml/cpu/conda_work/cpp/build
python ../scripts/cuda-memcheck.py -tool memcheck -exe ./test/prims
fi
gpuci_logger "Building doxygen C++ docs"
#Need to run in standard directory, not our artifact dir
unset LIBCUML_BUILD_DIR
$WORKSPACE/build.sh cppdocs -v
fi
if [ -n "\${CODECOV_TOKEN}" ]; then
codecov -t \$CODECOV_TOKEN
fi
return ${EXITCODE}