[docker-base-buster][docker-config-engine-buster] No longer install P…

…ython 2 (#6162) **- Why I did it** As part of migrating SONiC codebase from Python 2 to Python 3 **- How I did it** - No longer install Python 2 in docker-base-buster or docker-config-engine-buster. - Install Python 2 and pip2 in the following containers until we can completely eliminate it there: - docker-platform-monitor - docker-sonic-mgmt-framework - docker-sonic-vs - Pin pip2 version <21 where it is still temporarily needed, as pip version 21 will drop support for Python 2 - Also preform some other cleanup, ensuring that pip3, setuptools and wheel packages are installed in docker-base-buster, and then removing any attempts to re-install them in derived containers
sonic-net · Dec 26, 2020 · d40c9a1 · d40c9a1
1 parent 9a51505
commit d40c9a1
Show file tree

Hide file tree

Showing 16 changed files with 57 additions and 66 deletions.
diff --git a/dockers/docker-base-buster/Dockerfile.j2 b/dockers/docker-base-buster/Dockerfile.j2
@@ -44,9 +44,8 @@ RUN apt-get update &&        \
         less                 \
         perl                 \
         procps               \
-        python               \
-        python-pip           \
         python3              \
+        python3-distutils    \
         python3-pip          \
         rsyslog              \
         vim-tiny             \
@@ -57,7 +56,6 @@ RUN apt-get update &&        \
         lua-bitop            \
         lua-cjson            \
 # common dependencies
-        libpython2.7         \
         libdaemon0           \
         libdbus-1-3          \
         libjansson4          \
@@ -84,15 +82,16 @@ RUN dpkg -i redis-tools_6.0.6-1~bpo10+1_amd64.deb || apt-get install -f -y
 RUN rm redis-tools_6.0.6-1~bpo10+1_amd64.deb
 {% endif %}
 
-# Some Python packages require setuptools (or pkg_resources, which is supplied by setuptools)
-# and some require wheel
-RUN pip install setuptools==40.8.0
-RUN pip3 install setuptools==49.6.00
-RUN pip install wheel
-RUN pip3 install wheel
+# Upgrade pip via PyPI and uninstall the Debian version
+RUN pip3 install --upgrade pip
+RUN apt-get purge -y python3-pip
+
+# setuptools and wheel are necessary for installing some Python wheel packages
+RUN pip3 install --no-cache-dir setuptools==49.6.00
+RUN pip3 install --no-cache-dir wheel==0.35.1
 
 # For templating
-RUN pip2 install j2cli
+RUN pip3 install j2cli
 
 # Install supervisor
 RUN pip3 install supervisor==4.2.1

diff --git a/dockers/docker-base/Dockerfile.j2 b/dockers/docker-base/Dockerfile.j2
@@ -52,6 +52,9 @@ COPY ["etc/rsyslog.conf", "/etc/rsyslog.conf"]
 COPY ["etc/rsyslog.d/*", "/etc/rsyslog.d/"]
 COPY ["root/.vimrc", "/root/.vimrc"]
 
+RUN pip install --upgrade 'pip<21'
+RUN apt-get purge -y python-pip
+
 # Some Python packages require setuptools (or pkg_resources, which is supplied by setuptools)
 # and some require wheel
 RUN pip install setuptools==40.8.0

diff --git a/dockers/docker-config-engine-buster/Dockerfile.j2 b/dockers/docker-config-engine-buster/Dockerfile.j2
@@ -6,22 +6,15 @@ ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update         && \
     apt-get install -y        \
-        build-essential       \
-        python-pip            \
-        python3-pip           \
-        python-dev            \
-        python3-dev           \
         apt-utils             \
+        build-essential       \
+        python3-dev
+
 {%- if CONFIGURED_ARCH == "armhf" or CONFIGURED_ARCH == "arm64" %}
+RUN apt-get install -y        \
         libxslt-dev           \
-        libz-dev              \
+        libz-dev
 {%- endif %}
-        python-setuptools     \
-        python3-setuptools
-
-RUN pip3 install --upgrade pip
-RUN pip2 install --upgrade pip
-RUN apt-get purge -y python-pip python3-pip
 
 # For sonic-config-engine Python 3 package
 # Explicitly install pyangbind here, as pyangbind causes enum34 to be installed.
@@ -51,7 +44,6 @@ COPY ["files/swss_vars.j2", "/usr/share/sonic/templates/"]
 
 ## Clean up
 RUN apt-get purge -y               \
-        python-dev                 \
         python3-dev                \
         build-essential         && \
     apt-get clean -y            && \

diff --git a/dockers/docker-config-engine-stretch/Dockerfile.j2 b/dockers/docker-config-engine-stretch/Dockerfile.j2
@@ -8,15 +8,13 @@ RUN apt-get update         && \
     apt-get install -y        \
         # Dependencies for sonic-cfggen
         build-essential       \
-        python-pip            \
-        python-dev            \
+        python-dev
+
 {%- if CONFIGURED_ARCH == "armhf" or CONFIGURED_ARCH == "arm64" %}
+RUN apt-get install -y        \
         libxslt-dev           \
-        libz-dev              \
+        libz-dev
 {%- endif %}
-        python-setuptools
-
-RUN pip install --upgrade pip
 
 {% if docker_config_engine_stretch_debs.strip() %}
 # Copy locally-built Debian package dependencies
@@ -40,7 +38,6 @@ COPY ["files/swss_vars.j2", "/usr/share/sonic/templates/"]
 ## Clean up
 RUN apt-get purge -y               \
         build-essential            \
-        python-pip                 \
         python-dev              && \
     apt-get clean -y            && \
     apt-get autoclean -y        && \

diff --git a/dockers/docker-config-engine/Dockerfile.j2 b/dockers/docker-config-engine/Dockerfile.j2
@@ -6,11 +6,7 @@ ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update
 
 # Dependencies for sonic-cfggen
-RUN apt-get install -y build-essential python-pip python-dev
-
-RUN pip install --upgrade pip
-
-RUN apt-get purge -y python-pip
+RUN apt-get install -y build-essential python-dev
 
 {% if docker_config_engine_debs.strip() %}
 COPY \

diff --git a/dockers/docker-database/docker-database-init.sh b/dockers/docker-database/docker-database-init.sh
@@ -79,7 +79,7 @@ rm $db_cfg_file_tmp
 
 # copy dump.rdb file to each instance for restoration
 DUMPFILE=/var/lib/redis/dump.rdb
-redis_inst_list=`/usr/bin/python -c "import swsssdk; print(' '.join(swsssdk.SonicDBConfig.get_instancelist().keys()))"`
+redis_inst_list=`/usr/bin/python3 -c "import swsssdk; print(' '.join(swsssdk.SonicDBConfig.get_instancelist().keys()))"`
 for inst in $redis_inst_list
 do
     mkdir -p /var/lib/$inst

diff --git a/dockers/docker-database/supervisord.conf.j2 b/dockers/docker-database/supervisord.conf.j2
@@ -18,7 +18,7 @@ stdout_logfile=syslog
 stderr_logfile=syslog
 
 {% if INSTANCES %}
-{%     for redis_inst, redis_items in INSTANCES.iteritems() %}
+{%     for redis_inst, redis_items in INSTANCES.items() %}
 [program: {{ redis_inst }}]
 {% if redis_items['hostname'] != '127.0.0.1' and redis_inst != 'redis_chassis' %}
 {%- set LOOPBACK_IP = '127.0.0.1' -%}

diff --git a/dockers/docker-orchagent/Dockerfile.j2 b/dockers/docker-orchagent/Dockerfile.j2
@@ -21,7 +21,6 @@ RUN apt-get update &&       \
         ndppd               \
         # Needed for installing netifaces Python package
         build-essential     \
-        python-dev          \
         python3-dev
 
 {% if ( CONFIGURED_ARCH == "armhf" or CONFIGURED_ARCH == "arm64" ) %}
@@ -36,14 +35,6 @@ RUN ln -s -f /usr/bin/gcc-8 /usr/bin/arm-linux-gnueabihf-gcc
 RUN ln -s -f /usr/bin/gcc-8 /usr/bin/aarch64-linux-gnu-gcc
 {% endif %}
 
-RUN pip2 install \
-         scapy==2.4.2 \
-         setuptools \
-         pyroute2==0.5.3
-RUN pip2 install \
-         netifaces==0.10.7 \
-         monotonic==1.5
-
 # Dependencies of restore_neighbors.py
 RUN pip3 install \
          scapy==2.4.4 \
@@ -66,7 +57,6 @@ RUN apt-get remove -y gcc-8
 # Clean up
 RUN apt-get purge -y          \
         build-essential       \
-        python-dev            \
         python3-dev        && \
     apt-get clean -y       && \
     apt-get autoclean -y   && \

diff --git a/dockers/docker-platform-monitor/Dockerfile.j2 b/dockers/docker-platform-monitor/Dockerfile.j2
@@ -14,7 +14,7 @@ ENV IMAGE_VERSION=$image_version
 # Install required packages
 RUN apt-get update &&   \
     apt-get install -y  \
-        python-dev      \
+        build-essential \
         python3-dev     \
         ipmitool        \
         librrd8         \
@@ -26,6 +26,11 @@ RUN apt-get update &&   \
         i2c-tools       \
         python-jsonschema
 
+# TODO: Remove these lines once we no longer need Python 2
+RUN apt-get install -f -y python-dev python-pip
+RUN pip2 install --upgrade 'pip<21'
+RUN apt-get purge -y python-pip
+
 # On Arista devices, the sonic_platform wheel is not installed in the container.
 # Instead, the installation directory is mounted from the host OS. However, this method
 # doesn't ensure all dependencies are installed in the container. So here we
@@ -64,9 +69,13 @@ RUN pip3 install thrift==0.13.0
 {{ install_python_wheels(docker_platform_monitor_whls.split(' ')) }}
 {% endif %}
 
+
+# TODO: Remove this line once we no longer need Python 2
+RUN apt-get purge -y python-dev
+
 # Clean up
 RUN apt-get purge -y           \
-        python-dev             \
+        build-essential        \
         python3-dev         && \
     apt-get clean -y        && \
     apt-get autoclean -y    && \

diff --git a/dockers/docker-snmp/Dockerfile.j2 b/dockers/docker-snmp/Dockerfile.j2
@@ -40,12 +40,8 @@ ENV LC_CTYPE=en_US.UTF-8
 RUN sed -i '/^#.* en_US.* /s/^#//' /etc/locale.gen
 RUN locale-gen
 
-# Install up-to-date version of pip
-RUN pip3 install --no-cache-dir setuptools wheel
-
-# Install pyyaml dependency for use by some plugins
-# Install smbus dependency for use by some plugins
-RUN python3 -m pip install --no-cache-dir \
+# Install dependencies used by some plugins
+RUN pip3 install --no-cache-dir \
         hiredis                             \
         pyyaml                              \
         smbus

diff --git a/dockers/docker-sonic-mgmt-framework/Dockerfile.j2 b/dockers/docker-sonic-mgmt-framework/Dockerfile.j2
@@ -7,8 +7,14 @@ RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%s
 ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update && \
-    apt-get install -y g++ python-dev python3-dev libxml2
-
+    apt-get install -y g++ python3-dev libxml2
+
+# TODO: Remove these lines once we no longer need Python 2
+RUN apt-get install -f -y python-dev python-pip
+RUN pip2 install --upgrade 'pip<21'
+RUN apt-get purge -y python-pip
+RUN pip2 install setuptools==40.8.0
+RUN pip2 install wheel==0.35.1
 RUN pip2 install connexion==1.1.15 \
                 setuptools==21.0.0 \
                 grpcio-tools==1.20.0 \
@@ -40,7 +46,10 @@ COPY ["start.sh", "rest-server.sh", "/usr/bin/"]
 COPY ["mgmt_vars.j2", "/usr/share/sonic/templates/"]
 COPY ["supervisord.conf", "/etc/supervisor/conf.d/"]
 
-RUN apt-get remove -y g++ python-dev python3-dev
+# TODO: Remove this line once we no longer need Python 2
+RUN apt-get purge -y python-dev
+
+RUN apt-get remove -y g++ python3-dev
 RUN apt-get clean -y; apt-get autoclean -y; apt-get autoremove -y
 RUN rm -rf /debs
 

diff --git a/platform/vs/docker-sonic-vs/Dockerfile.j2 b/platform/vs/docker-sonic-vs/Dockerfile.j2
@@ -42,13 +42,13 @@ RUN apt-get install -y net-tools \
                        python-scapy \
                        conntrack \
                        iptables \
-                       python3-pip \
                        jq \
                        libzmq5 \
                        # For installing Python m2crypto package
                        # (these can be uninstalled after installation)
                        build-essential \
                        python-dev \
+                       python-pip \
                        python3-dev \
                        libssl-dev \
                        swig \
@@ -73,8 +73,10 @@ RUN dpkg -i redis-tools_6.0.6-1~bpo10+1_amd64.deb redis-server_6.0.6-1~bpo10+1_a
 RUN rm redis-tools_6.0.6-1~bpo10+1_amd64.deb redis-server_6.0.6-1~bpo10+1_amd64.deb
 {% endif %}
 
-RUN pip2 install setuptools
-RUN pip3 install setuptools
+RUN pip2 install --upgrade 'pip<21'
+RUN apt-get purge -y python-pip
+RUN pip2 install setuptools==40.8.0
+RUN pip2 install wheel==0.33.6
 RUN pip2 install py2_ipaddress
 RUN pip2 install six
 RUN pip2 install pyroute2==0.5.3 netifaces==0.10.7

diff --git a/rules/docker-config-engine-buster.mk b/rules/docker-config-engine-buster.mk
@@ -4,11 +4,8 @@ DOCKER_CONFIG_ENGINE_BUSTER = docker-config-engine-buster.gz
 $(DOCKER_CONFIG_ENGINE_BUSTER)_PATH = $(DOCKERS_PATH)/docker-config-engine-buster
 
 $(DOCKER_CONFIG_ENGINE_BUSTER)_DEPENDS += $(LIBSWSSCOMMON) $(PYTHON3_SWSSCOMMON)
-$(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SWSSSDK_PY2)
 $(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SWSSSDK_PY3)
-$(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SONIC_PY_COMMON_PY2)
 $(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SONIC_PY_COMMON_PY3)
-$(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SONIC_CONFIG_ENGINE_PY2)
 $(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SONIC_CONFIG_ENGINE_PY3)
 $(DOCKER_CONFIG_ENGINE_BUSTER)_LOAD_DOCKERS += $(DOCKER_BASE_BUSTER)
 $(DOCKER_CONFIG_ENGINE_BUSTER)_FILES += $(SWSS_VARS_TEMPLATE)

diff --git a/rules/docker-platform-monitor.mk b/rules/docker-platform-monitor.mk
@@ -14,6 +14,7 @@ $(DOCKER_PLATFORM_MONITOR)_PYTHON_WHEELS += $(SWSSSDK_PY2)
 $(DOCKER_PLATFORM_MONITOR)_PYTHON_WHEELS += $(SWSSSDK_PY3)
 $(DOCKER_PLATFORM_MONITOR)_PYTHON_WHEELS += $(SONIC_PY_COMMON_PY2)
 $(DOCKER_PLATFORM_MONITOR)_PYTHON_WHEELS += $(SONIC_PY_COMMON_PY3)
+$(DOCKER_PLATFORM_MONITOR)_PYTHON_WHEELS += $(SONIC_CONFIG_ENGINE_PY2)
 $(DOCKER_PLATFORM_MONITOR)_PYTHON_WHEELS += $(SONIC_PLATFORM_API_PY2)
 $(DOCKER_PLATFORM_MONITOR)_PYTHON_WHEELS += $(SONIC_PLATFORM_API_PY3)
 $(DOCKER_PLATFORM_MONITOR)_PYTHON_WHEELS += $(SONIC_LEDD)

diff --git a/sonic-slave-buster/Dockerfile.j2 b/sonic-slave-buster/Dockerfile.j2
@@ -370,7 +370,7 @@ RUN export VERSION=1.14.2 \
  && rm go$VERSION.linux-*.tar.gz
 
 RUN pip3 install --upgrade pip
-RUN pip2 install --upgrade pip
+RUN pip2 install --upgrade 'pip<21'
 RUN apt-get purge -y python-pip python3-pip
 
 # For building Python packages

diff --git a/sonic-slave-stretch/Dockerfile.j2 b/sonic-slave-stretch/Dockerfile.j2
@@ -339,7 +339,7 @@ RUN export VERSION=1.14.2 \
  && rm go$VERSION.linux-*.tar.gz
 
 RUN pip3 install --upgrade pip
-RUN pip2 install --upgrade pip
+RUN pip2 install --upgrade 'pip<21'
 RUN apt-get purge -y python-pip python3-pip
 
 # For p4 build