Fix wheel relocation issues (#2777)

* [DEBUG] Check wheel relocation issues * Call delocate on Mac * Use yum instead of conda * Do not copy ffmpeg dylibs * Linux dry run * Do not download FFmpeg on Mac * Install bzip2 * Restore FFmpeg on Windows * Remove ffmpeg temporarily * Do not remove dependencies * Disable FFmpeg temporarily on Linux wheels * Test relocation in Linux * Add docstring * Call relocation script * Minor error correction * Import auditwheel only on Linux * Restore ffmpeg again on Windows * Return *device * Try fix Windows * Fix clang-format * Start windows patchwork * Copy all DLLs * Disable FFmpeg on Windows for now * Bugfix Co-authored-by: Francisco Massa <fvsmassa@gmail.com>
pytorch · Oct 16, 2020 · 643fdc3 · 643fdc3
1 parent 02c2537
commit 643fdc3
Show file tree

Hide file tree

Showing 4 changed files with 457 additions and 33 deletions.
diff --git a/packaging/build_wheel.sh b/packaging/build_wheel.sh
@@ -17,19 +17,20 @@ if [[ "$(uname)" == Darwin || "$OSTYPE" == "msys" ]]; then
     bin_path=$(dirname $python_exec)
     env_path=$(dirname $bin_path)
     if [[ "$(uname)" == Darwin ]]; then
-        # Include LibPNG
-        cp "$env_path/lib/libpng16.dylib" torchvision
-        # Include LibJPEG
-        cp "$env_path/lib/libjpeg.dylib" torchvision
+        # Install delocate to relocate the required binaries
+        pip_install delocate
     else
         cp "$bin_path/Library/bin/libpng16.dll" torchvision
         cp "$bin_path/Library/bin/libjpeg.dll" torchvision
     fi
 else
-    # Include LibPNG
-    cp "/usr/lib64/libpng.so" torchvision
-    # Include LibJPEG
-    cp "/usr/lib64/libjpeg.so" torchvision
+    # Install auditwheel to get some inspection utilities
+    pip_install auditwheel
+
+    # Point to custom libraries
+    export LD_LIBRARY_PATH=$(pwd)/ext_libraries/lib:$LD_LIBRARY_PATH
+    export TORCHVISION_INCLUDE=$(pwd)/ext_libraries/include
+    export TORCHVISION_LIBRARY=$(pwd)/ext_libraries/lib
 fi
 
 
@@ -38,3 +39,20 @@ if [[ "$OSTYPE" == "msys" ]]; then
 else
     IS_WHEEL=1 python setup.py bdist_wheel
 fi
+
+
+if [[ "$(uname)" == Darwin ]]; then
+    pushd dist/
+    python_exec="$(which python)"
+    bin_path=$(dirname $python_exec)
+    env_path=$(dirname $bin_path)
+    for whl in *.whl; do
+        DYLD_LIBRARY_PATH="$env_path/lib/:$DYLD_LIBRARY_PATH" delocate-wheel -v $whl
+    done
+else
+    if [[ "$OSTYPE" == "msys" ]]; then
+        "$script_dir/windows/internal/vc_env_helper.bat" python $script_dir/wheel/relocate.py
+    else
+        LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH" python $script_dir/wheel/relocate.py
+    fi
+fi
diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash
@@ -184,8 +184,8 @@ setup_wheel_python() {
     # Install libpng from Anaconda (defaults)
     conda install libpng jpeg -y
   else
-    # Install native CentOS libPNG
-    yum install -y libpng-devel libjpeg-turbo-devel
+    # Install native CentOS libJPEG, LAME, freetype and GnuTLS
+    yum install -y libjpeg-turbo-devel lame freetype gnutls
     case "$PYTHON_VERSION" in
       2.7)
         if [[ -n "$UNICODE_ABI" ]]; then
@@ -203,7 +203,13 @@ setup_wheel_python() {
         exit 1
         ;;
     esac
-    export PATH="/opt/python/$python_abi/bin:$PATH"
+    # Download all the dependencies required to compile image and video_reader
+    # extensions
+
+    mkdir -p ext_libraries
+    pushd ext_libraries
+    popd
+    export PATH="/opt/python/$python_abi/bin:$(pwd)/ext_libraries/bin:$PATH"
   fi
 }
 
@@ -353,10 +359,8 @@ setup_junit_results_folder() {
 
 
 download_copy_ffmpeg() {
-  mkdir ffmpeg_tmp
-  cd ffmpeg_tmp
   if [[ "$OSTYPE" == "msys" ]]; then
-    # conda install -yq ffmpeg -c pytorch
+    # conda install -yq ffmpeg=4.2 -c pytorch
     # curl -L -q https://anaconda.org/pytorch/ffmpeg/4.3/download/win-64/ffmpeg-4.3-ha925a31_0.tar.bz2 --output ffmpeg-4.3-ha925a31_0.tar.bz2
     # bzip2 --decompress --stdout ffmpeg-4.3-ha925a31_0.tar.bz2 | tar -x --file=-
     # cp Library/bin/*.dll ../torchvision
@@ -365,24 +369,15 @@ download_copy_ffmpeg() {
     if [[ "$(uname)" == Darwin ]]; then
       conda install -yq ffmpeg=4.2 -c pytorch
       conda install -yq wget
-      wget -q https://anaconda.org/pytorch/ffmpeg/4.2/download/osx-64/ffmpeg-4.2-h0a44026_0.tar.bz2
-      tar -xjvf ffmpeg-4.2-h0a44026_0.tar.bz2
-      for f in lib/*.dylib; do
-        if [[ $f =~ ([a-z])+\.dylib ]]; then
-          cp $f ../torchvision
-        fi
-      done
     else
-      wget -q https://anaconda.org/pytorch/ffmpeg/4.2/download/linux-64/ffmpeg-4.2-hf484d3e_0.tar.bz2
-      tar -xjvf ffmpeg-4.2-hf484d3e_0.tar.bz2
-      cp lib/*.so ../torchvision
-      cp -r lib/* /usr/lib
-      cp -r bin/* /usr/bin
-      cp -r include/* /usr/include
-      ldconfig
-      which ffmpeg
+      # pushd ext_libraries
+      # wget -q https://anaconda.org/pytorch/ffmpeg/4.2/download/linux-64/ffmpeg-4.2-hf484d3e_0.tar.bz2
+      # tar -xjvf ffmpeg-4.2-hf484d3e_0.tar.bz2
+      # rm -rf ffmpeg-4.2-hf484d3e_0.tar.bz2
+      # ldconfig
+      # which ffmpeg
+      # popd
+      echo "FFmpeg is disabled currently on Linux"
     fi
   fi
-  cd ..
-  rm -rf ffmpeg_tmp
 }