[RFC] Pytest environment improvements

docker/bash.sh

@@ -89,6 +89,7 @@ ${DOCKER_BINARY} run --rm --pid=host\
     -e "CI_BUILD_GROUP=$(id -g -n)" \
     -e "CI_BUILD_GID=$(id -g)" \
     -e "PYTHONPATH=python:topi/python"\
+    -e "CI_PYTEST_ADD_OPTIONS=$CI_PYTEST_ADD_OPTIONS" \
     ${CUDA_ENV}\
     ${CI_DOCKER_EXTRA_PARAMS[@]} \
     ${DOCKER_IMAGE_NAME}\

docker/build.sh

@@ -162,6 +162,7 @@ ${DOCKER_BINARY} run --rm --pid=host \
     -e "CI_BUILD_UID=$(id -u)" \
     -e "CI_BUILD_GROUP=$(id -g -n)" \
     -e "CI_BUILD_GID=$(id -g)" \
+    -e "CI_PYTEST_ADD_OPTIONS=$CI_PYTEST_ADD_OPTIONS" \
     ${CUDA_ENV}\
     ${CI_DOCKER_EXTRA_PARAMS[@]} \
     ${DOCKER_IMG_NAME} \

docker/with_the_same_user

@@ -41,6 +41,7 @@ getent passwd "${CI_BUILD_UID}" || adduser --gid "${CI_BUILD_GID}" --uid "${CI_B
     --gecos "${CI_BUILD_USER} (generated by with_the_same_user script)" \
     --disabled-password --home "${CI_BUILD_HOME}" --quiet "${CI_BUILD_USER}"
 usermod -a -G sudo "${CI_BUILD_USER}"
+# This is a grotesque hack to get PYTEST_ADD_OPTS available to all task scripts.
 echo "${CI_BUILD_USER} ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/90-nopasswd-sudo
 
 if [[ ! -z $CUDA_VISIBLE_DEVICES ]]; then

docs/contribute/pull_request.rst

@@ -118,3 +118,6 @@ If you want to run a single test:
   rm -rf python/tvm/*.pyc python/tvm/*/*.pyc python/tvm/*/*/*.pyc
 
   TVM_FFI=ctypes python -m pytest -v tests/python/unittest/test_pass_storage_rewrite.py
+
+  #Additionally if you want to run a single test, for example test_all_elemwise inside a file.
+  TVM_FFI=ctypes python -m pytest -v -k "test_all_elemwise" tests/python/frontend/tflite/test_forward.py

tests/scripts/setup-pytest-env.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -u
+set -e
+export TVM_PATH=`pwd`
+export PYTHONPATH=python:topi/python
+export PYTEST_ADDOPTS="-v $CI_PYTEST_ADD_OPTIONS"

tests/scripts/task_python_docs.sh

@@ -19,6 +19,7 @@
 set -e
 set -u
 
+source tests/scripts/setup-pytest-env.sh
 # cleanup old states
 rm -rf docs/_build
 mkdir -p docs/_build/html

tests/scripts/task_python_frontend.sh

@@ -19,7 +19,7 @@
 set -e
 set -u
 
-export PYTHONPATH=python:topi/python
+source tests/scripts/setup-pytest-env.sh
 # to avoid openblas threading error
 export TVM_BIND_THREADS=0
 export OMP_NUM_THREADS=1
@@ -30,28 +30,28 @@ find . -type f -path "*.pyc" | xargs rm -f
 make cython3
 
 echo "Running relay TFLite frontend test..."
-python3 -m pytest -v tests/python/frontend/tflite
+python3 -m pytest tests/python/frontend/tflite
 
 echo "Running relay MXNet frontend test..."
-python3 -m pytest -v tests/python/frontend/mxnet
+python3 -m pytest tests/python/frontend/mxnet
 
 echo "Running relay Keras frontend test..."
-python3 -m pytest -v tests/python/frontend/keras
+python3 -m pytest tests/python/frontend/keras
 
 echo "Running relay ONNX frontend test..."
-python3 -m pytest -v tests/python/frontend/onnx
+python3 -m pytest tests/python/frontend/onnx
 
 echo "Running relay CoreML frontend test..."
-python3 -m pytest -v tests/python/frontend/coreml
+python3 -m pytest tests/python/frontend/coreml
 
 echo "Running relay Tensorflow frontend test..."
-python3 -m pytest -v tests/python/frontend/tensorflow
+python3 -m pytest tests/python/frontend/tensorflow
 
 echo "Running relay caffe2 frontend test..."
-python3 -m pytest -v tests/python/frontend/caffe2
+python3 -m pytest tests/python/frontend/caffe2
 
 echo "Running relay DarkNet frontend test..."
-python3 -m pytest -v tests/python/frontend/darknet
+python3 -m pytest tests/python/frontend/darknet
 
 echo "Running relay PyTorch frontend test..."
-python3 -m pytest -v tests/python/frontend/pytorch
+python3 -m pytest tests/python/frontend/pytorch

tests/scripts/task_python_integration.sh

@@ -19,7 +19,8 @@
 set -e
 set -u
 
-export PYTHONPATH=`pwd`/python:`pwd`/topi/python:`pwd`/apps/extension/python
+source tests/scripts/setup-pytest-env.sh
+export PYTHONPATH=${PYTHONPATH}:${TVM_PATH}/apps/extension/python
 export LD_LIBRARY_PATH="build:${LD_LIBRARY_PATH:-}"
 export TVM_BIND_THREADS=0
 export TVM_NUM_THREADS=2
@@ -42,26 +43,26 @@ rm -rf lib
 make
 cd ../..
 
-TVM_FFI=cython python3 -m pytest -v apps/extension/tests
-TVM_FFI=ctypes python3 -m pytest -v apps/extension/tests
+TVM_FFI=cython python3 -m pytest apps/extension/tests
+TVM_FFI=ctypes python3 -m pytest apps/extension/tests
 
 # Test dso plugin
 cd apps/dso_plugin_module
 rm -rf lib
 make
 cd ../..
-TVM_FFI=cython python3 -m pytest -v apps/dso_plugin_module
-TVM_FFI=ctypes python3 -m pytest -v apps/dso_plugin_module
+TVM_FFI=cython python3 -m pytest apps/dso_plugin_module
+TVM_FFI=ctypes python3 -m pytest apps/dso_plugin_module
 
 # Do not enable TensorFlow op
 # TVM_FFI=cython sh prepare_and_test_tfop_module.sh
 # TVM_FFI=ctypes sh prepare_and_test_tfop_module.sh
 
-TVM_FFI=ctypes python3 -m pytest -v tests/python/integration
-TVM_FFI=ctypes python3 -m pytest -v tests/python/contrib
+TVM_FFI=ctypes python3 -m pytest tests/python/integration
+TVM_FFI=ctypes python3 -m pytest tests/python/contrib
 
-TVM_FFI=ctypes python3 -m pytest -v tests/python/relay
+TVM_FFI=ctypes python3 -m pytest tests/python/relay
 
 # Do not enable OpenGL
-# TVM_FFI=cython python -m pytest -v tests/webgl
-# TVM_FFI=ctypes python3 -m pytest -v tests/webgl
+# TVM_FFI=cython python -m pytest tests/webgl
+# TVM_FFI=ctypes python3 -m pytest tests/webgl

tests/scripts/task_python_nightly.sh

@@ -19,12 +19,12 @@
 set -e
 set -u
 
-export PYTHONPATH=python:topi/python
+source tests/scripts/setup-pytest-env.sh
 
 # Rebuild cython
 make cython3
 
 # cleanup pycache
 find . -type f -path "*.pyc" | xargs rm -f
 
-python3 -m pytest -v topi/tests/python/nightly
+python3 -m pytest topi/tests/python/nightly

tests/scripts/task_python_topi.sh

@@ -19,12 +19,11 @@
 set -e
 set -u
 
-export PYTHONPATH=python:topi/python
-
+source tests/scripts/setup-pytest-env.sh
 # Rebuild cython
 make cython3
 
 # cleanup pycache
 find . -type f -path "*.pyc" | xargs rm -f
 
-python3 -m pytest -v topi/tests/python
+python3 -m pytest topi/tests/python

tests/scripts/task_python_unittest.sh

@@ -19,11 +19,11 @@
 set -e
 set -u
 
-export PYTHONPATH=python:topi/python
+source tests/scripts/setup-pytest-env.sh
 
 # cleanup pycache
 find . -type f -path "*.pyc" | xargs rm -f
 
-TVM_FFI=ctypes python3 -m pytest -v tests/python/unittest
+TVM_FFI=ctypes python3 -m pytest tests/python/unittest
 make cython3
-TVM_FFI=cython python3 -m pytest -v tests/python/unittest
+TVM_FFI=cython python3 -m pytest tests/python/unittest

tests/scripts/task_python_vta_fsim.sh

@@ -19,8 +19,8 @@
 set -e
 set -u
 
-export TVM_PATH=`pwd`
-export PYTHONPATH=${TVM_PATH}/python:${TVM_PATH}/vta/python:${TVM_PATH}/topi/python
+source tests/scripts/setup-pytest-env.sh
+export PYTHONPATH=${TVM_PATH}/python:${TVM_PATH}/vta/python
 export VTA_HW_PATH=`pwd`/3rdparty/vta-hw
 
 # cleanup pycache
@@ -36,8 +36,8 @@ cp ${VTA_HW_PATH}/config/fsim_sample.json ${VTA_HW_PATH}/config/vta_config.json
 
 # Run unit tests in functional/fast simulator
 echo "Running unittest in fsim..."
-python3 -m pytest -v ${TVM_PATH}/vta/tests/python/unittest
+python3 -m pytest ${TVM_PATH}/vta/tests/python/unittest
 
 # Run unit tests in functional/fast simulator
 echo "Running integration test in fsim..."
-python3 -m pytest -v ${TVM_PATH}/vta/tests/python/integration
+python3 -m pytest ${TVM_PATH}/vta/tests/python/integration

tests/scripts/task_python_vta_tsim.sh

@@ -19,8 +19,8 @@
 set -e
 set -u
 
-export TVM_PATH=`pwd`
-export PYTHONPATH=${TVM_PATH}/python:${TVM_PATH}/vta/python:${TVM_PATH}/topi/python
+source tests/scripts/setup-pytest-env.sh
+export PYTHONPATH=${PYTHONPATH}:${TVM_PATH}/vta/python
 export VTA_HW_PATH=`pwd`/3rdparty/vta-hw
 
 # cleanup pycache
@@ -51,11 +51,11 @@ make -C ${VTA_HW_PATH}/hardware/chisel USE_THREADS=0 lib
 
 # Run unit tests in cycle accurate simulator
 echo "Running unittest in tsim..."
-python3 -m pytest -v ${TVM_PATH}/vta/tests/python/unittest
+python3 -m pytest ${TVM_PATH}/vta/tests/python/unittest
 
 # Run unit tests in cycle accurate simulator
 echo "Running integration test in tsim..."
-python3 -m pytest -v ${TVM_PATH}/vta/tests/python/integration
+python3 -m pytest ${TVM_PATH}/vta/tests/python/integration
 
 # Reset default fsim simulation
 cp ${VTA_HW_PATH}/config/fsim_sample.json ${VTA_HW_PATH}/config/vta_config.json