diff --git a/docker/Dockerfile.ci_arm b/docker/Dockerfile.ci_arm
index c19f1ff5a4c13..b7aaba47f4316 100644
--- a/docker/Dockerfile.ci_arm
+++ b/docker/Dockerfile.ci_arm
@@ -20,8 +20,11 @@
 
 FROM ubuntu:18.04
 
+COPY utils/apt-install-and-clear.sh /usr/local/bin/apt-install-and-clear
+
 RUN apt-get update --fix-missing
-RUN apt-get install -y ca-certificates gnupg2
+
+RUN apt-install-and-clear -y ca-certificates gnupg2
 
 COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh
 RUN bash /install/ubuntu_install_core.sh
diff --git a/docker/Dockerfile.ci_cpu b/docker/Dockerfile.ci_cpu
index 45943334a06ff..2dc075d29b139 100644
--- a/docker/Dockerfile.ci_cpu
+++ b/docker/Dockerfile.ci_cpu
@@ -18,6 +18,8 @@
 # CI docker CPU env
 FROM ubuntu:18.04
 
+COPY utils/apt-install-and-clear.sh /usr/local/bin/apt-install-and-clear
+
 RUN apt-get update --fix-missing
 
 COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh
diff --git a/docker/Dockerfile.ci_gpu b/docker/Dockerfile.ci_gpu
index e0d1997de729b..22c372cc70b00 100644
--- a/docker/Dockerfile.ci_gpu
+++ b/docker/Dockerfile.ci_gpu
@@ -19,6 +19,8 @@
 # tag: v0.60
 FROM nvidia/cuda:11.0.3-cudnn8-devel-ubuntu18.04
 
+COPY utils/apt-install-and-clear.sh /usr/local/bin/apt-install-and-clear
+
 # Per https://forums.developer.nvidia.com/t/notice-cuda-linux-repository-key-rotation/212772
 # we need to add a new GPG key before running apt update.
 RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub
@@ -55,7 +57,7 @@ COPY install/ubuntu_install_sphinx.sh /install/ubuntu_install_sphinx.sh
 RUN bash /install/ubuntu_install_sphinx.sh
 
 # Enable doxygen for c++ doc build
-RUN apt-get update && apt-get install -y doxygen libprotobuf-dev protobuf-compiler
+RUN apt-get update && apt-install-and-clear -y doxygen libprotobuf-dev protobuf-compiler
 
 COPY install/ubuntu_install_java.sh /install/ubuntu_install_java.sh
 RUN bash /install/ubuntu_install_java.sh
diff --git a/docker/Dockerfile.ci_hexagon b/docker/Dockerfile.ci_hexagon
index ddca5c6c2e666..cf7407c2ab05d 100644
--- a/docker/Dockerfile.ci_hexagon
+++ b/docker/Dockerfile.ci_hexagon
@@ -19,8 +19,11 @@
 # tag: v0.02
 FROM tvmcihexagon/ci-hexagon-base:v0.02_SDK4.5.0.3
 
+COPY utils/apt-install-and-clear.sh /usr/local/bin/apt-install-and-clear
+
 RUN apt-get update --fix-missing
-RUN apt-get install -y ca-certificates gnupg2 libxml2-dev
+
+RUN apt-install-and-clear -y ca-certificates gnupg2 libxml2-dev
 
 COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh
 RUN bash /install/ubuntu_install_core.sh
diff --git a/docker/Dockerfile.ci_i386 b/docker/Dockerfile.ci_i386
index 61ba064ff3f1a..0b6d8d28c4d7c 100644
--- a/docker/Dockerfile.ci_i386
+++ b/docker/Dockerfile.ci_i386
@@ -20,7 +20,11 @@
 
 FROM i386/ubuntu:18.04
 
-RUN apt-get update --fix-missing && apt-get install -y ca-certificates
+COPY utils/apt-install-and-clear.sh /usr/local/bin/apt-install-and-clear
+
+RUN apt-get update --fix-missing
+
+RUN apt-install-and-clear -y ca-certificates
 
 COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh
 RUN bash /install/ubuntu_install_core.sh
diff --git a/docker/Dockerfile.ci_jekyll b/docker/Dockerfile.ci_jekyll
index 5d3cf86dd6f52..f6912a8a9651d 100644
--- a/docker/Dockerfile.ci_jekyll
+++ b/docker/Dockerfile.ci_jekyll
@@ -19,6 +19,9 @@
 # tag: v0.50
 FROM ubuntu:16.04
 
-RUN apt-get update && apt-get install -y sudo wget
-RUN apt-get update && apt-get install -y ruby-full build-essential zlib1g-dev
+COPY utils/apt-install-and-clear.sh /usr/local/bin/apt-install-and-clear
+
+RUN apt-get update
+
+RUN apt-install-and-clear -y sudo wget ruby-full build-essential zlib1g-dev
 RUN gem install jekyll bundler
diff --git a/docker/Dockerfile.ci_lint b/docker/Dockerfile.ci_lint
index 1d0c984c6190d..437ea71bd4bea 100644
--- a/docker/Dockerfile.ci_lint
+++ b/docker/Dockerfile.ci_lint
@@ -20,9 +20,11 @@
 # tag: v0.60
 FROM ubuntu:18.04
 
+COPY utils/apt-install-and-clear.sh /usr/local/bin/apt-install-and-clear
+
 RUN apt-get update --fix-missing
 
-RUN apt-get update && apt-get install -y wget git sudo make parallel
+RUN apt-install-and-clear -y wget git sudo make parallel
 
 COPY install/ubuntu1804_install_python.sh /install/ubuntu1804_install_python.sh
 RUN bash /install/ubuntu1804_install_python.sh
@@ -30,7 +32,7 @@ RUN bash /install/ubuntu1804_install_python.sh
 # Globally disable pip cache
 RUN pip config set global.no-cache-dir false
 
-RUN apt-get update && apt-get install -y doxygen graphviz curl shellcheck
+RUN apt-get update && apt-install-and-clear -y doxygen graphviz curl shellcheck
 
 RUN pip3 install cpplint pylint==2.4.4 mypy==0.902 black==22.3.0 flake8==3.9.2 blocklint==0.2.3 jinja2==3.0.3
 
diff --git a/docker/Dockerfile.ci_qemu b/docker/Dockerfile.ci_qemu
index 28bfd8962de53..851a3c520e3ac 100644
--- a/docker/Dockerfile.ci_qemu
+++ b/docker/Dockerfile.ci_qemu
@@ -19,6 +19,8 @@
 # tag: v0.62
 FROM ubuntu:18.04
 
+COPY utils/apt-install-and-clear.sh /usr/local/bin/apt-install-and-clear
+
 RUN apt-get update --fix-missing
 
 COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh
diff --git a/docker/Dockerfile.ci_wasm b/docker/Dockerfile.ci_wasm
index 1c7d3eb59b41c..49435b4f3d479 100644
--- a/docker/Dockerfile.ci_wasm
+++ b/docker/Dockerfile.ci_wasm
@@ -16,6 +16,8 @@
 # under the License.
 FROM ubuntu:18.04
 
+COPY utils/apt-install-and-clear.sh /usr/local/bin/apt-install-and-clear
+
 RUN apt-get update --fix-missing
 
 COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh
diff --git a/docker/Dockerfile.conda_cpu b/docker/Dockerfile.conda_cpu
index d2779afbdaf37..1ca7a743b0ef7 100644
--- a/docker/Dockerfile.conda_cpu
+++ b/docker/Dockerfile.conda_cpu
@@ -17,7 +17,11 @@
 
 FROM ubuntu:16.04
 
-RUN apt-get update --fix-missing && apt-get install -y bzip2 wget sudo binutils git
+COPY utils/apt-install-and-clear.sh /usr/local/bin/apt-install-and-clear
+
+RUN apt-get update --fix-missing
+
+RUN apt-install-and-clear -y bzip2 wget sudo binutils git
 
 COPY install/ubuntu_install_conda.sh /install/ubuntu_install_conda.sh
 RUN bash /install/ubuntu_install_conda.sh
diff --git a/docker/Dockerfile.conda_cuda100 b/docker/Dockerfile.conda_cuda100
index 7705c8548b523..8e5a1a1b7fa51 100644
--- a/docker/Dockerfile.conda_cuda100
+++ b/docker/Dockerfile.conda_cuda100
@@ -17,7 +17,11 @@
 
 FROM nvidia/cuda:10.0-devel-ubuntu16.04
 
-RUN apt-get update --fix-missing && apt-get install -y bzip2 wget sudo binutils git
+COPY utils/apt-install-and-clear.sh /usr/local/bin/apt-install-and-clear
+
+RUN apt-get update --fix-missing
+
+RUN apt-install-and-clear -y bzip2 wget sudo binutils git
 
 RUN  wget -q http://developer.download.nvidia.com/compute/redist/cudnn/v7.6.0/cudnn-10.0-linux-x64-v7.6.0.64.tgz && \
     tar --no-same-owner -xzf cudnn-10.0-linux-x64-v7.6.0.64.tgz -C /usr/local && \
diff --git a/docker/Dockerfile.conda_cuda90 b/docker/Dockerfile.conda_cuda90
index 3721674381410..cfc109200e019 100644
--- a/docker/Dockerfile.conda_cuda90
+++ b/docker/Dockerfile.conda_cuda90
@@ -17,7 +17,11 @@
 
 FROM nvidia/cuda:9.0-devel-ubuntu16.04
 
-RUN apt-get update --fix-missing && apt-get install -y bzip2 wget sudo binutils git
+COPY utils/apt-install-and-clear.sh /usr/local/bin/apt-install-and-clear
+
+RUN apt-get update --fix-missing
+
+RUN apt-install-and-clear -y bzip2 wget sudo binutils git
 
 RUN  wget -q http://developer.download.nvidia.com/compute/redist/cudnn/v7.6.0/cudnn-9.0-linux-x64-v7.6.0.64.tgz && \
     tar --no-same-owner -xzf cudnn-9.0-linux-x64-v7.6.0.64.tgz -C /usr/local && \
diff --git a/docker/Dockerfile.demo_android b/docker/Dockerfile.demo_android
index 8236075a3d3c6..e66fb3aa3cfaf 100644
--- a/docker/Dockerfile.demo_android
+++ b/docker/Dockerfile.demo_android
@@ -18,6 +18,8 @@
 # Minimum docker image for demo purposes
 FROM ubuntu:18.04
 
+COPY utils/apt-install-and-clear.sh /usr/local/bin/apt-install-and-clear
+
 RUN apt-get update --fix-missing
 
 COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh
diff --git a/docker/Dockerfile.demo_cpu b/docker/Dockerfile.demo_cpu
index c4397e02f50b1..d081f26423c15 100644
--- a/docker/Dockerfile.demo_cpu
+++ b/docker/Dockerfile.demo_cpu
@@ -19,6 +19,8 @@
 # prebuilt-image: tvmai/demo-cpu
 FROM tlcpack/ci-cpu:v0.55
 
+COPY utils/apt-install-and-clear.sh /usr/local/bin/apt-install-and-clear
+
 # Jupyter notebook.
 RUN pip3 install matplotlib Image Pillow jupyter[notebook]
 
diff --git a/docker/Dockerfile.demo_gpu b/docker/Dockerfile.demo_gpu
index c3b973f138079..4ef6b0c29cbcf 100644
--- a/docker/Dockerfile.demo_gpu
+++ b/docker/Dockerfile.demo_gpu
@@ -20,6 +20,8 @@
 # tag: v0.54
 FROM tlcpack/ci-gpu:v0.55
 
+COPY utils/apt-install-and-clear.sh /usr/local/bin/apt-install-and-clear
+
 # Jupyter notebook.
 RUN pip3 install matplotlib Image "Pillow<7" jupyter[notebook]
 
diff --git a/docker/Dockerfile.demo_opencl b/docker/Dockerfile.demo_opencl
index 2f534d8b5b5ce..7f497b7807264 100644
--- a/docker/Dockerfile.demo_opencl
+++ b/docker/Dockerfile.demo_opencl
@@ -21,6 +21,8 @@
 
 FROM ubuntu:18.04
 
+COPY utils/apt-install-and-clear.sh /usr/local/bin/apt-install-and-clear
+
 RUN echo "Labelling this image"
 LABEL Description="Docker image for TVM built with OpenCL & OpenGL support"
 
@@ -30,29 +32,25 @@ RUN apt-get update
 RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
 
 RUN echo "Installing utility libraries"
-RUN apt-get install -y apt-utils sudo
-RUN apt-get install -y cmake g++ llvm
-RUN apt-get install -y git
-# make wget unzip libtinfo-dev libz-dev libcurl4-openssl-dev
-RUN apt-get install -y libopenblas-dev
+RUN apt-install-and-clear -y apt-utils sudo cmake g++ llvm git libopenblas-dev
 
 # RUN echo "Installing gtest"
-# RUN apt-get install -y libgtest-dev
+# RUN apt-install-and-clear -y libgtest-dev
 # RUN cd /usr/src/gtest && cmake CMakeLists.txt && make && cp *.a /usr/lib
 
 RUN echo "Installing Python"
-RUN apt-get install -y python3-dev python3-pip
+RUN apt-install-and-clear -y python3-dev python3-pip
 RUN pip3 install setuptools numpy pytest cython decorator scipy tornado psutil xgboost
 
 RUN echo "Installing Jupyter notebook"
 RUN pip3 install matplotlib Image "Pillow<7" jupyter[notebook]
 
 RUN echo "Installing OpenCL libraries"
-RUN apt-get install -y libviennacl-dev mesa-opencl-icd ocl-icd-opencl-dev clinfo
-RUN apt-get install -y libclblas-dev libclfft-dev libclsparse-dev
+RUN apt-install-and-clear -y libviennacl-dev mesa-opencl-icd ocl-icd-opencl-dev clinfo
+RUN apt-install-and-clear -y libclblas-dev libclfft-dev libclsparse-dev
 
 RUN echo "Installing OpenGL libraries"
-RUN apt-get install -y libcogl-dev libegl1 libgles1 libglfw3-dev
+RUN apt-install-and-clear -y libcogl-dev libegl1 libgles1 libglfw3-dev
 # libglew-dev
 
 RUN echo "Upgrading dependencies"
diff --git a/docker/Dockerfile.demo_rocm b/docker/Dockerfile.demo_rocm
index c336be41934fd..1dd7d1bf119af 100644
--- a/docker/Dockerfile.demo_rocm
+++ b/docker/Dockerfile.demo_rocm
@@ -18,6 +18,8 @@
 # Demo docker for ROCm
 FROM ubuntu:18.04
 
+COPY utils/apt-install-and-clear.sh /usr/local/bin/apt-install-and-clear
+
 COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh
 RUN bash /install/ubuntu_install_core.sh
 
diff --git a/docker/Dockerfile.demo_vitis_ai b/docker/Dockerfile.demo_vitis_ai
index c38ccaf0e5914..445f74ced982c 100644
--- a/docker/Dockerfile.demo_vitis_ai
+++ b/docker/Dockerfile.demo_vitis_ai
@@ -18,6 +18,8 @@
 # Main Vitis AI docker env
 FROM xilinx/vitis-ai:1.4.916
 
+COPY utils/apt-install-and-clear.sh /usr/local/bin/apt-install-and-clear
+
 RUN apt-get update --fix-missing
 
 COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh
diff --git a/docker/Dockerfile.docs b/docker/Dockerfile.docs
index 840094b4d0cbc..9fe90a7302c0b 100644
--- a/docker/Dockerfile.docs
+++ b/docker/Dockerfile.docs
@@ -18,6 +18,9 @@
 FROM ubuntu:18.04
 
 # Base scripts
+
+COPY utils/apt-install-and-clear.sh /usr/local/bin/apt-install-and-clear
+
 RUN apt-get update --fix-missing
 
 COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh
@@ -36,7 +39,7 @@ COPY install/ubuntu_install_sphinx.sh /install/ubuntu_install_sphinx.sh
 RUN bash /install/ubuntu_install_sphinx.sh
 
 # Enable doxygen for c++ doc build
-RUN apt-get update && apt-get install -y doxygen libprotobuf-dev protobuf-compiler
+RUN apt-get update && apt-install-and-clear -y doxygen libprotobuf-dev protobuf-compiler
 
 COPY install/ubuntu_install_java.sh /install/ubuntu_install_java.sh
 RUN bash /install/ubuntu_install_java.sh
diff --git a/docker/install/ubuntu1804_install_clang_format.sh b/docker/install/ubuntu1804_install_clang_format.sh
index e830433bb0391..ffc885810aedb 100755
--- a/docker/install/ubuntu1804_install_clang_format.sh
+++ b/docker/install/ubuntu1804_install_clang_format.sh
@@ -26,4 +26,4 @@ echo deb-src http://apt.llvm.org/bionic/ llvm-toolchain-bionic-10 main\
      >> /etc/apt/sources.list.d/llvm.list
 
 wget -q -O - http://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
-apt-get update && apt-get install -y clang-format-10
+apt-get update && apt-install-and-clear -y clang-format-10
diff --git a/docker/install/ubuntu1804_install_llvm.sh b/docker/install/ubuntu1804_install_llvm.sh
index b4640aa9ae6ec..cc821ab6a41b5 100755
--- a/docker/install/ubuntu1804_install_llvm.sh
+++ b/docker/install/ubuntu1804_install_llvm.sh
@@ -53,7 +53,7 @@ echo deb-src http://apt.llvm.org/bionic/ llvm-toolchain-bionic main\
 
 wget -q -O - http://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add -
 
-apt-get update && apt-get install -y \
+apt-get update && apt-install-and-clear -y \
      llvm-9 llvm-10 llvm-11 llvm-12 llvm-13 \
      clang-9 libclang-9-dev \
      clang-10 libclang-10-dev \
diff --git a/docker/install/ubuntu1804_install_python.sh b/docker/install/ubuntu1804_install_python.sh
index 94d316199db58..2cdddbd451a61 100755
--- a/docker/install/ubuntu1804_install_python.sh
+++ b/docker/install/ubuntu1804_install_python.sh
@@ -31,8 +31,7 @@ trap cleanup 0
 # Install python and pip. Don't modify this to add Python package dependencies,
 # instead modify install_python_package.sh
 apt-get update
-apt-get install -y software-properties-common
-apt-get install -y python3.7 python3.7-dev python3-pip
+apt-install-and-clear -y software-properties-common python3.7 python3.7-dev python3-pip
 update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 1
 
 # Pin pip and setuptools versions
diff --git a/docker/install/ubuntu1804_install_python_venv.sh b/docker/install/ubuntu1804_install_python_venv.sh
index 5dc5efea76f6d..3f0fb3ee89716 100755
--- a/docker/install/ubuntu1804_install_python_venv.sh
+++ b/docker/install/ubuntu1804_install_python_venv.sh
@@ -22,8 +22,7 @@ set -o pipefail
 
 # install python and pip, don't modify this, modify install_python_package.sh
 apt-get update
-apt-get install -y software-properties-common
-apt-get install -y python3.7-dev python3-setuptools python3.7-venv
+apt-install-and-clear -y software-properties-common python3.7-dev python3-setuptools python3.7-venv
 
 python3 -mvenv /opt/tvm-venv
 
diff --git a/docker/install/ubuntu2004_install_python.sh b/docker/install/ubuntu2004_install_python.sh
index 5b87a74061fb5..8b14ea07907b3 100755
--- a/docker/install/ubuntu2004_install_python.sh
+++ b/docker/install/ubuntu2004_install_python.sh
@@ -30,8 +30,7 @@ trap cleanup 0
 # Install python and pip. Don't modify this to add Python package dependencies,
 # instead modify install_python_package.sh
 apt-get update
-apt-get install -y software-properties-common
-apt-get install -y python3.8 python3.8-dev python3-pip
+apt-install-and-clear -y software-properties-common python3.8 python3.8-dev python3-pip
 update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1
 
 # Pin pip and setuptools versions
diff --git a/docker/install/ubuntu_download_arm_compute_lib_binaries.sh b/docker/install/ubuntu_download_arm_compute_lib_binaries.sh
index 1f0b34c11165d..051a94b71c0c7 100755
--- a/docker/install/ubuntu_download_arm_compute_lib_binaries.sh
+++ b/docker/install/ubuntu_download_arm_compute_lib_binaries.sh
@@ -22,7 +22,8 @@ architecture_type=$(uname -i)
 # Depending on the architecture selected to compile for,
 # you may need to install an alternative cross-compiler.
 if [ "$architecture_type" != "aarch64" ]; then
-  apt-get update && apt-get install -y --no-install-recommends \
+  apt-get update
+  apt-install-and-clear -y --no-install-recommends \
     g++-aarch64-linux-gnu \
     gcc-aarch64-linux-gnu
 fi
diff --git a/docker/install/ubuntu_install_arduino.sh b/docker/install/ubuntu_install_arduino.sh
index bb27b56b995dc..107b452f8d3f3 100755
--- a/docker/install/ubuntu_install_arduino.sh
+++ b/docker/install/ubuntu_install_arduino.sh
@@ -21,7 +21,7 @@ set -u
 set -o pipefail
 
 export DEBIAN_FRONTEND=noninteractive
-apt-get install -y ca-certificates
+apt-install-and-clear -y ca-certificates
 
 ARDUINO_CLI_VERSION="0.21.1"
 # Install arduino-cli
diff --git a/docker/install/ubuntu_install_caffe.sh b/docker/install/ubuntu_install_caffe.sh
index ab71eab54a2d6..c37bfb764935c 100755
--- a/docker/install/ubuntu_install_caffe.sh
+++ b/docker/install/ubuntu_install_caffe.sh
@@ -21,7 +21,7 @@ set -euxo pipefail
 apt-get update --fix-missing
 
 # # Install dependencies
-apt-get install -y --no-install-recommends protobuf-compiler \
+apt-install-and-clear -y --no-install-recommends protobuf-compiler \
     libprotobuf-dev libhdf5-serial-dev libopenblas-dev libgflags-dev libgoogle-glog-dev
 
 
diff --git a/docker/install/ubuntu_install_core.sh b/docker/install/ubuntu_install_core.sh
index 5593d61ea5c90..eba4318f07727 100755
--- a/docker/install/ubuntu_install_core.sh
+++ b/docker/install/ubuntu_install_core.sh
@@ -21,7 +21,7 @@ set -u
 set -o pipefail
 
 # install libraries for building c++ core on ubuntu
-apt-get update && apt-get install -y --no-install-recommends \
+apt-get update && apt-install-and-clear -y --no-install-recommends \
     apt-transport-https \
     ca-certificates \
     cmake \
diff --git a/docker/install/ubuntu_install_ethosn_driver_stack.sh b/docker/install/ubuntu_install_ethosn_driver_stack.sh
index 873486e965624..e7878d8e4ba7a 100755
--- a/docker/install/ubuntu_install_ethosn_driver_stack.sh
+++ b/docker/install/ubuntu_install_ethosn_driver_stack.sh
@@ -37,7 +37,7 @@ trap cleanup 0
 # Ubuntu 16.04 dependencies
 apt-get update
 
-apt-get install -y \
+apt-install-and-clear -y \
     bsdmainutils \
     build-essential \
     cmake \
diff --git a/docker/install/ubuntu_install_ethosu_driver_stack.sh b/docker/install/ubuntu_install_ethosu_driver_stack.sh
index cbb55c9c0de04..d34445e2e80f8 100755
--- a/docker/install/ubuntu_install_ethosu_driver_stack.sh
+++ b/docker/install/ubuntu_install_ethosu_driver_stack.sh
@@ -38,8 +38,7 @@ trap cleanup 0
 
 # Ubuntu 18.04 dependencies
 apt-get update
-
-apt-get install -y \
+apt-install-and-clear -y \
     bsdmainutils \
     build-essential \
     cpp \
diff --git a/docker/install/ubuntu_install_golang.sh b/docker/install/ubuntu_install_golang.sh
index 5b05b3886b05c..4e8e743266c16 100755
--- a/docker/install/ubuntu_install_golang.sh
+++ b/docker/install/ubuntu_install_golang.sh
@@ -22,6 +22,4 @@ set -o pipefail
 
 #install the necessary dependancies for golang build
 apt-get update
-apt-get install -y golang-1.10-go
-apt-get install -y golang-1.10-doc
-apt-get install -y golint
+apt-install-and-clear -y golang-1.10-go golang-1.10-doc golint
diff --git a/docker/install/ubuntu_install_java.sh b/docker/install/ubuntu_install_java.sh
index 7e8ee39d00226..d0ced98d89f16 100755
--- a/docker/install/ubuntu_install_java.sh
+++ b/docker/install/ubuntu_install_java.sh
@@ -19,7 +19,8 @@
 set -o errexit -o nounset
 set -o pipefail
 
-apt-get update && apt-get install -y openjdk-8-jdk maven
+apt-get update
+apt-install-and-clear -y openjdk-8-jdk maven
 arch=$(uname -m)
 jre_arch="unknown"
 case $arch in
diff --git a/docker/install/ubuntu_install_llvm.sh b/docker/install/ubuntu_install_llvm.sh
index 6616bfc5eb2b6..4da9d9f4441ed 100755
--- a/docker/install/ubuntu_install_llvm.sh
+++ b/docker/install/ubuntu_install_llvm.sh
@@ -19,7 +19,7 @@
 set -euxo pipefail
 
 apt-get update
-apt-get install -y gnupg
+apt-install-and-clear -y gnupg
 
 echo deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-4.0 main\
      >> /etc/apt/sources.list.d/llvm.list
@@ -47,4 +47,4 @@ echo deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial main\
      >> /etc/apt/sources.list.d/llvm.list
 
 wget -q -O - http://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add -
-apt-get install -y llvm-4.0 llvm-9 llvm-8 llvm-7 clang-9 libclang-9-dev clang-8 libclang-8-dev clang-7 libclang-7-dev
+apt-install-and-clear -y llvm-4.0 llvm-9 llvm-8 llvm-7 clang-9 libclang-9-dev clang-8 libclang-8-dev clang-7 libclang-7-dev
diff --git a/docker/install/ubuntu_install_nnpack.sh b/docker/install/ubuntu_install_nnpack.sh
index 744f76a162bbb..e4a37f56f7eba 100755
--- a/docker/install/ubuntu_install_nnpack.sh
+++ b/docker/install/ubuntu_install_nnpack.sh
@@ -20,7 +20,7 @@ set -e
 set -u
 set -o pipefail
 
-apt-get update && apt-get install -y --no-install-recommends git cmake python-setuptools
+apt-get update && apt-install-and-clear -y --no-install-recommends git cmake python-setuptools
 
 git clone https://github.com/Maratyszcza/NNPACK NNPACK
 git clone https://github.com/Maratyszcza/pthreadpool  NNPACK/pthreadpool
diff --git a/docker/install/ubuntu_install_nodejs.sh b/docker/install/ubuntu_install_nodejs.sh
index de3ba31747b6d..2bb8a115a0c44 100755
--- a/docker/install/ubuntu_install_nodejs.sh
+++ b/docker/install/ubuntu_install_nodejs.sh
@@ -24,10 +24,9 @@ apt-get update
 # Please do not remove 'curl' package installation from here, as this
 # script runs in some images (e.g. ci_lint) that keep a very mininal
 # set of packages installed by default.
-apt-get install -y curl
+apt-install-and-clear -y curl
 
 # The node install script fetched and executed here will update the
-# apt source list, hence the second apt-get update is necessary.
+# apt source list, hence the second apt-get update --fix-missing is necessary.
 curl -s -S -L https://deb.nodesource.com/setup_14.x | bash -
-apt-get update
-apt-get install -y nodejs
+apt-install-and-clear -y nodejs
\ No newline at end of file
diff --git a/docker/install/ubuntu_install_opencl.sh b/docker/install/ubuntu_install_opencl.sh
index ca6101675307c..705f4a65eedb1 100755
--- a/docker/install/ubuntu_install_opencl.sh
+++ b/docker/install/ubuntu_install_opencl.sh
@@ -21,10 +21,9 @@ set -u
 set -o pipefail
 
 # Install OpenCL runtime in nvidia docker.
-apt-get update && apt-get install -y --no-install-recommends \
+apt-get update && apt-install-and-clear -y --no-install-recommends \
         ocl-icd-opencl-dev \
-        clinfo && \
-    rm -rf /var/lib/apt/lists/*
+        clinfo
 
 mkdir -p /etc/OpenCL/vendors && \
     echo "libnvidia-opencl.so.1" > /etc/OpenCL/vendors/nvidia.icd
diff --git a/docker/install/ubuntu_install_papi.sh b/docker/install/ubuntu_install_papi.sh
index 713312ee58c96..62f8c281e770e 100755
--- a/docker/install/ubuntu_install_papi.sh
+++ b/docker/install/ubuntu_install_papi.sh
@@ -23,7 +23,7 @@ set -o pipefail
 apt-get update --fix-missing
 
 # deps
-apt-get install -y linux-tools-common linux-tools-generic
+apt-install-and-clear -y linux-tools-common linux-tools-generic
 
 cd /
 git clone https://bitbucket.org/icl/papi.git
diff --git a/docker/install/ubuntu_install_python.sh b/docker/install/ubuntu_install_python.sh
index b71398ad5fc86..c8856f299ba03 100755
--- a/docker/install/ubuntu_install_python.sh
+++ b/docker/install/ubuntu_install_python.sh
@@ -22,14 +22,14 @@ set -o pipefail
 
 # install python and pip, don't modify this, modify install_python_package.sh
 apt-get update
-apt-get install -y python-dev
+apt-install-and-clear -y python-dev
 
 # python 3.6
-apt-get install -y software-properties-common
+apt-install-and-clear -y software-properties-common
 
 add-apt-repository -y ppa:deadsnakes/ppa
 apt-get update
-apt-get install -y python-pip python-dev python3.6 python3.6-dev
+apt-install-and-clear -y python-pip python-dev python3.6 python3.6-dev
 
 rm -f /usr/bin/python3 && ln -s /usr/bin/python3.6 /usr/bin/python3
 
diff --git a/docker/install/ubuntu_install_qemu.sh b/docker/install/ubuntu_install_qemu.sh
index 6682795b0fd8c..b34ac018d94cd 100755
--- a/docker/install/ubuntu_install_qemu.sh
+++ b/docker/install/ubuntu_install_qemu.sh
@@ -79,4 +79,4 @@ make -j${num_cores}
 sudo make install
 
 # For debugging with qemu
-apt-get -y install libpython3.8
+apt-install-and-clear -y install libpython3.8
diff --git a/docker/install/ubuntu_install_redis.sh b/docker/install/ubuntu_install_redis.sh
index 8678c2050100a..72049f7b9a30a 100755
--- a/docker/install/ubuntu_install_redis.sh
+++ b/docker/install/ubuntu_install_redis.sh
@@ -20,6 +20,6 @@ set -e
 set -u
 set -o pipefail
 
-apt-get update && apt-get install -y redis-server
+apt-get update && apt-install-and-clear -y redis-server
 pip3 install \
     xgboost==1.4.2
diff --git a/docker/install/ubuntu_install_rocm.sh b/docker/install/ubuntu_install_rocm.sh
index 2f28356da3c86..15cb1e143ac46 100755
--- a/docker/install/ubuntu_install_rocm.sh
+++ b/docker/install/ubuntu_install_rocm.sh
@@ -23,8 +23,6 @@ set -o pipefail
 # Install ROCm cross compilation toolchain.
 wget -qO - https://repo.radeon.com/rocm/rocm.gpg.key | sudo apt-key add -
 echo 'deb [arch=amd64] https://repo.radeon.com/rocm/apt/4.3/ ubuntu main' | sudo tee /etc/apt/sources.list.d/rocm.list
-apt-get update && apt-get install -y \
+apt-get update && apt-install-and-clear -y \
     rocm-dev \
-    lld-12 && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
+    lld-12
diff --git a/docker/install/ubuntu_install_sbt.sh b/docker/install/ubuntu_install_sbt.sh
index 713faad03a43c..d27c9b7352fe2 100755
--- a/docker/install/ubuntu_install_sbt.sh
+++ b/docker/install/ubuntu_install_sbt.sh
@@ -22,7 +22,7 @@ set -o pipefail
 
 # The https:// source added below required an apt https transport
 # support.
-apt-get update && apt-get install -y apt-transport-https
+apt-get update && apt-install-and-clear -y apt-transport-https
 
 # Install the necessary dependencies for sbt
 echo "deb https://repo.scala-sbt.org/scalasbt/debian all main" | sudo tee /etc/apt/sources.list.d/sbt.list
@@ -31,4 +31,4 @@ apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2EE0EA64E40A89B84B2
 
 # Note: The settings in vta/hardware/chisel/project/build.properties
 # file determines required sbt version.
-apt-get update && apt-get install -y sbt=1.1.1
+apt-get update && apt-install-and-clear -y sbt=1.1.1
diff --git a/docker/install/ubuntu_install_tensorflow_aarch64.sh b/docker/install/ubuntu_install_tensorflow_aarch64.sh
index 8d5b6765deb05..82f8b28bf3c61 100755
--- a/docker/install/ubuntu_install_tensorflow_aarch64.sh
+++ b/docker/install/ubuntu_install_tensorflow_aarch64.sh
@@ -19,7 +19,7 @@
 set -euxo pipefail
 
 # Build dependencies
-apt-get install -y --no-install-recommends libhdf5-dev
+apt-install-and-clear -y --no-install-recommends libhdf5-dev
 
 # We're only using the TensorFlow wheel snapshot here as the
 # h5py wheel tries to use the wrong .so file
diff --git a/docker/install/ubuntu_install_verilator.sh b/docker/install/ubuntu_install_verilator.sh
index 9361bba70277c..fe89b30758725 100755
--- a/docker/install/ubuntu_install_verilator.sh
+++ b/docker/install/ubuntu_install_verilator.sh
@@ -24,7 +24,7 @@ set -o pipefail
 version="4.104"
 
 # Install dependencies
-apt-get update && apt-get install -y autoconf g++ flex bison
+apt-get update && apt-install-and-clear -y autoconf g++ flex bison
 
 # Install Verilator
 wget "https://github.com/verilator/verilator/archive/v$version.tar.gz"
diff --git a/docker/install/ubuntu_install_vitis_ai_core.sh b/docker/install/ubuntu_install_vitis_ai_core.sh
index 09e7aaea931ba..48980d2e7ba24 100755
--- a/docker/install/ubuntu_install_vitis_ai_core.sh
+++ b/docker/install/ubuntu_install_vitis_ai_core.sh
@@ -24,12 +24,11 @@ export PYXIR_HOME=/opt/pyxir
 mkdir "$PYXIR_HOME"
 
 # install libraries for building Vitis-AI on ubuntu
-apt-get update && apt-get install -y \
+apt-get update && apt-install-and-clear -y \
     graphviz \
     gnupg2 \
     gpg-agent \
-    gcc-aarch64-linux-gnu \
-    && rm -rf /var/lib/apt/lists/*
+    gcc-aarch64-linux-gnu
 
 
 . $VAI_ROOT/conda/etc/profile.d/conda.sh
diff --git a/docker/install/ubuntu_install_vulkan.sh b/docker/install/ubuntu_install_vulkan.sh
index b7d2d4672b0cb..78cd4143f83eb 100755
--- a/docker/install/ubuntu_install_vulkan.sh
+++ b/docker/install/ubuntu_install_vulkan.sh
@@ -22,5 +22,5 @@ set -o pipefail
 
 wget -qO - http://packages.lunarg.com/lunarg-signing-key-pub.asc | apt-key add -
 wget -qO /etc/apt/sources.list.d/lunarg-vulkan-1.2.135-xenial.list http://packages.lunarg.com/vulkan/1.2.135/lunarg-vulkan-1.2.135-xenial.list
-apt update
-apt install -y vulkan-sdk
+apt-get update
+apt-install-and-clear -y vulkan-sdk
diff --git a/docker/install/ubuntu_install_wasmtime.sh b/docker/install/ubuntu_install_wasmtime.sh
index 0a086a019b94b..b93527135c976 100755
--- a/docker/install/ubuntu_install_wasmtime.sh
+++ b/docker/install/ubuntu_install_wasmtime.sh
@@ -19,7 +19,7 @@
 set -euxo pipefail
 
 # install wasmtime (note: requires ubuntu_install_rust.sh to run first)
-apt-get install -y --no-install-recommends libc6-dev-i386
+apt-install-and-clear -y --no-install-recommends libc6-dev-i386
 export WASMTIME_HOME=/opt/wasmtime
 curl https://wasmtime.dev/install.sh -sSf | bash
 export PATH="${WASMTIME_HOME}/bin:${PATH}"
diff --git a/docker/install/ubuntu_install_zephyr.sh b/docker/install/ubuntu_install_zephyr.sh
index 1237f91a41525..1755079a12764 100755
--- a/docker/install/ubuntu_install_zephyr.sh
+++ b/docker/install/ubuntu_install_zephyr.sh
@@ -26,7 +26,7 @@ export TZ=Etc/UTC
 sudo ln -snf /usr/share/zoneinfo/$TZ /etc/localtime
 echo $TZ > /etc/timezone
 
-sudo apt-get install -y --no-install-recommends \
+sudo apt-install-and-clear -y --no-install-recommends \
      libsdl2-dev ca-certificates gnupg software-properties-common wget \
      git cmake ninja-build gperf \
      ccache dfu-util device-tree-compiler wget \
@@ -40,7 +40,7 @@ echo deb https://apt.kitware.com/ubuntu/ bionic main\
      >> /etc/apt/sources.list.d/kitware.list
 sudo apt-get update
 
-sudo apt-get install -y cmake
+sudo apt-install-and-clear -y cmake
 
 pip3 install west
 
diff --git a/docker/utils/apt-install-and-clear.sh b/docker/utils/apt-install-and-clear.sh
new file mode 100755
index 0000000000000..1840c17b37bbb
--- /dev/null
+++ b/docker/utils/apt-install-and-clear.sh
@@ -0,0 +1,20 @@
+#!/bin/bash -e
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+apt-get install $@ && apt-get clean
+
diff --git a/tests/lint/docker-format.sh b/tests/lint/docker-format.sh
new file mode 100755
index 0000000000000..98fc8286c8da2
--- /dev/null
+++ b/tests/lint/docker-format.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+TVM_HOME="$(git rev-parse --show-toplevel)"
+DOCKER_DIR="$TVM_HOME/docker"
+
+if grep -R --exclude='apt-install-and-clear.sh' "apt install" $DOCKER_DIR; then
+  echo "Found \"apt install\" in docker file."
+  exit 1
+fi
+
+if grep -R --exclude='apt-install-and-clear.sh' "apt-get install" $DOCKER_DIR; then
+  echo "Found \"apt-get install\" in docker file."
+  exit 1
+fi
+
+exit 0
\ No newline at end of file
diff --git a/tests/scripts/task_lint.sh b/tests/scripts/task_lint.sh
index 8fbba52662de8..80cfc00ff7be4 100755
--- a/tests/scripts/task_lint.sh
+++ b/tests/scripts/task_lint.sh
@@ -74,6 +74,9 @@ function shard2 {
 
   echo "Rust check..."
   tests/lint/rust_format.sh
+
+  echo "Docker check..."
+  tests/lint/docker-format.sh
 }