Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix wheel relocation issues #2777

Merged
merged 30 commits into from
Oct 14, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
b78e94e
[DEBUG] Check wheel relocation issues
andfoy Oct 8, 2020
4ad8921
Call delocate on Mac
andfoy Oct 8, 2020
76f3b0a
Use yum instead of conda
andfoy Oct 8, 2020
d1f1bc5
Do not copy ffmpeg dylibs
andfoy Oct 8, 2020
6888d14
Linux dry run
andfoy Oct 8, 2020
c9627b1
Do not download FFmpeg on Mac
andfoy Oct 8, 2020
90611ef
Install bzip2
andfoy Oct 8, 2020
4dedd60
Merge branch 'master' into debug_wheels
andfoy Oct 9, 2020
3364818
Restore FFmpeg on Windows
andfoy Oct 12, 2020
3b45810
Merge branch 'debug_wheels' of github.com:andfoy/vision into debug_wh…
andfoy Oct 12, 2020
cc307fd
Merge branch 'master' into debug_wheels
andfoy Oct 12, 2020
be54605
Remove ffmpeg temporarily
andfoy Oct 12, 2020
a5b5f31
Merge branch 'debug_wheels' of github.com:andfoy/vision into debug_wh…
andfoy Oct 12, 2020
ffc6e2f
Do not remove dependencies
andfoy Oct 12, 2020
5f439f8
Disable FFmpeg temporarily on Linux wheels
andfoy Oct 12, 2020
690b872
Test relocation in Linux
andfoy Oct 13, 2020
ba1cac4
Add docstring
andfoy Oct 13, 2020
104ed58
Call relocation script
andfoy Oct 13, 2020
db0d453
Minor error correction
andfoy Oct 13, 2020
a1b0823
Import auditwheel only on Linux
andfoy Oct 13, 2020
09e151b
Restore ffmpeg again on Windows
andfoy Oct 13, 2020
39e5e10
Merge branch 'master' into debug_wheels
andfoy Oct 13, 2020
154522f
Return *device
andfoy Oct 13, 2020
8d53aa8
Merge branch 'debug_wheels' of github.com:andfoy/vision into debug_wh…
andfoy Oct 13, 2020
2111d86
Try fix Windows
fmassa Oct 13, 2020
f5917e2
Fix clang-format
fmassa Oct 13, 2020
a6edd4d
Start windows patchwork
andfoy Oct 13, 2020
d08e3b9
Copy all DLLs
andfoy Oct 13, 2020
c7f0f12
Disable FFmpeg on Windows for now
fmassa Oct 14, 2020
7654534
Bugfix
fmassa Oct 14, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 26 additions & 8 deletions packaging/build_wheel.sh
Original file line number Diff line number Diff line change
Expand Up @@ -17,19 +17,20 @@ if [[ "$(uname)" == Darwin || "$OSTYPE" == "msys" ]]; then
bin_path=$(dirname $python_exec)
env_path=$(dirname $bin_path)
if [[ "$(uname)" == Darwin ]]; then
# Include LibPNG
cp "$env_path/lib/libpng16.dylib" torchvision
# Include LibJPEG
cp "$env_path/lib/libjpeg.dylib" torchvision
# Install delocate to relocate the required binaries
pip_install delocate
else
cp "$bin_path/Library/bin/libpng16.dll" torchvision
cp "$bin_path/Library/bin/libjpeg.dll" torchvision
fi
else
# Include LibPNG
cp "/usr/lib64/libpng.so" torchvision
# Include LibJPEG
cp "/usr/lib64/libjpeg.so" torchvision
# Install auditwheel to get some inspection utilities
pip_install auditwheel

# Point to custom libraries
export LD_LIBRARY_PATH=$(pwd)/ext_libraries/lib:$LD_LIBRARY_PATH
export TORCHVISION_INCLUDE=$(pwd)/ext_libraries/include
export TORCHVISION_LIBRARY=$(pwd)/ext_libraries/lib
fi

download_copy_ffmpeg
Expand All @@ -39,3 +40,20 @@ if [[ "$OSTYPE" == "msys" ]]; then
else
IS_WHEEL=1 python setup.py bdist_wheel
fi


if [[ "$(uname)" == Darwin ]]; then
pushd dist/
python_exec="$(which python)"
bin_path=$(dirname $python_exec)
env_path=$(dirname $bin_path)
for whl in *.whl; do
DYLD_LIBRARY_PATH="$env_path/lib/:$DYLD_LIBRARY_PATH" delocate-wheel -v $whl
done
else
if [[ "$OSTYPE" == "msys" ]]; then
"$script_dir/windows/internal/vc_env_helper.bat" python $script_dir/wheel/relocate.py
else
LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH" python $script_dir/wheel/relocate.py
fi
fi
41 changes: 18 additions & 23 deletions packaging/pkg_helpers.bash
Original file line number Diff line number Diff line change
Expand Up @@ -184,8 +184,8 @@ setup_wheel_python() {
# Install libpng from Anaconda (defaults)
conda install libpng jpeg -y
else
# Install native CentOS libPNG
yum install -y libpng-devel libjpeg-turbo-devel
# Install native CentOS libJPEG, LAME, freetype and GnuTLS
yum install -y libjpeg-turbo-devel lame freetype gnutls
case "$PYTHON_VERSION" in
2.7)
if [[ -n "$UNICODE_ABI" ]]; then
Expand All @@ -203,7 +203,13 @@ setup_wheel_python() {
exit 1
;;
esac
export PATH="/opt/python/$python_abi/bin:$PATH"
# Download all the dependencies required to compile image and video_reader
# extensions

mkdir -p ext_libraries
pushd ext_libraries
popd
export PATH="/opt/python/$python_abi/bin:$(pwd)/ext_libraries/bin:$PATH"
fi
}

Expand Down Expand Up @@ -353,10 +359,8 @@ setup_junit_results_folder() {


download_copy_ffmpeg() {
mkdir ffmpeg_tmp
cd ffmpeg_tmp
if [[ "$OSTYPE" == "msys" ]]; then
# conda install -yq ffmpeg -c pytorch
# conda install -yq ffmpeg=4.2 -c pytorch
# curl -L -q https://anaconda.org/pytorch/ffmpeg/4.3/download/win-64/ffmpeg-4.3-ha925a31_0.tar.bz2 --output ffmpeg-4.3-ha925a31_0.tar.bz2
# bzip2 --decompress --stdout ffmpeg-4.3-ha925a31_0.tar.bz2 | tar -x --file=-
# cp Library/bin/*.dll ../torchvision
Expand All @@ -365,24 +369,15 @@ download_copy_ffmpeg() {
if [[ "$(uname)" == Darwin ]]; then
conda install -yq ffmpeg=4.2 -c pytorch
conda install -yq wget
wget -q https://anaconda.org/pytorch/ffmpeg/4.2/download/osx-64/ffmpeg-4.2-h0a44026_0.tar.bz2
tar -xjvf ffmpeg-4.2-h0a44026_0.tar.bz2
for f in lib/*.dylib; do
if [[ $f =~ ([a-z])+\.dylib ]]; then
cp $f ../torchvision
fi
done
else
wget -q https://anaconda.org/pytorch/ffmpeg/4.2/download/linux-64/ffmpeg-4.2-hf484d3e_0.tar.bz2
tar -xjvf ffmpeg-4.2-hf484d3e_0.tar.bz2
cp lib/*.so ../torchvision
cp -r lib/* /usr/lib
cp -r bin/* /usr/bin
cp -r include/* /usr/include
ldconfig
which ffmpeg
# pushd ext_libraries
# wget -q https://anaconda.org/pytorch/ffmpeg/4.2/download/linux-64/ffmpeg-4.2-hf484d3e_0.tar.bz2
# tar -xjvf ffmpeg-4.2-hf484d3e_0.tar.bz2
# rm -rf ffmpeg-4.2-hf484d3e_0.tar.bz2
# ldconfig
# which ffmpeg
# popd
echo "FFmpeg is disabled currently on Linux"
fi
fi
cd ..
rm -rf ffmpeg_tmp
}
Loading