Merge pull request #568 from vphilippon/unicode-fix

Fix unicode encoding errors inside pip (Copy of #567)

CHANGELOG.md

@@ -13,6 +13,7 @@ when `--allow-unsafe` was not set. ([#517](https://github.com/jazzband/pip-tools
 - Fixed bug where some primary dependencies were annotated with "via" info comments. ([#542](https://github.com/jazzband/pip-tools/pull/542)). Thanks @quantus
 - Fixed bug where pkg-resources would be removed by pip-sync in Ubuntu. ([#555](https://github.com/jazzband/pip-tools/pull/555)). Thanks @cemsbr
 - Fixed bug where the resolver would sometime not stabilize on requirements specifying extras. ([#566](https://github.com/jazzband/pip-tools/pull/566)). Thanks @vphilippon
+- Fixed an unicode encoding error when distribution package contains non-ASCII file names ([#567](https://github.com/jazzband/pip-tools/pull/567)). Thanks @suutari
 
 # 1.9.0 (2017-04-12)
 

piptools/_compat/tempfile.py

@@ -1,6 +1,5 @@
 # coding: utf-8
-from __future__ import (absolute_import, division, print_function,
-                        unicode_literals)
+from __future__ import absolute_import, division, print_function
 
 import os as _os
 import sys as _sys

piptools/repositories/pypi.py

@@ -17,7 +17,7 @@
 
 from ..cache import CACHE_DIR
 from ..exceptions import NoCandidateFound
-from ..utils import (is_pinned_requirement, lookup_table,
+from ..utils import (fs_str, is_pinned_requirement, lookup_table,
                      make_install_requirement, pip_version_info)
 from .base import BaseRepository
 
@@ -83,16 +83,16 @@ def __init__(self, pip_options, session):
 
         # Setup file paths
         self.freshen_build_caches()
-        self._download_dir = os.path.join(CACHE_DIR, 'pkgs')
-        self._wheel_download_dir = os.path.join(CACHE_DIR, 'wheels')
+        self._download_dir = fs_str(os.path.join(CACHE_DIR, 'pkgs'))
+        self._wheel_download_dir = fs_str(os.path.join(CACHE_DIR, 'wheels'))
 
     def freshen_build_caches(self):
         """
         Start with fresh build/source caches.  Will remove any old build
         caches from disk automatically.
         """
-        self._build_dir = TemporaryDirectory('build')
-        self._source_dir = TemporaryDirectory('source')
+        self._build_dir = TemporaryDirectory(fs_str('build'))
+        self._source_dir = TemporaryDirectory(fs_str('source'))
 
     @property
     def build_dir(self):

piptools/utils.py

@@ -58,7 +58,9 @@ def make_install_requirement(name, version, extras, constraint=False):
         # Sort extras for stability
         extras_string = "[{}]".format(",".join(sorted(extras)))
 
-    return InstallRequirement.from_line('{}{}=={}'.format(name, extras_string, str(version)), constraint=constraint)
+    return InstallRequirement.from_line(
+        str('{}{}=={}'.format(name, extras_string, version)),
+        constraint=constraint)
 
 
 def format_requirement(ireq, marker=None):
@@ -208,3 +210,30 @@ def dedup(iterable):
     order-reserved.
     """
     return iter(OrderedDict.fromkeys(iterable))
+
+
+def fs_str(string):
+    """
+    Convert given string to a correctly encoded filesystem string.
+
+    On Python 2, if the input string is unicode, converts it to bytes
+    encoded with the filesystem encoding.
+
+    On Python 3 returns the string as is, since Python 3 uses unicode
+    paths and the input string shouldn't be bytes.
+
+    >>> fs_str(u'some path component/Something')
+    'some path component/Something'
+    >>> assert isinstance(fs_str('whatever'), str)
+    >>> assert isinstance(fs_str(u'whatever'), str)
+
+    :type string: str|unicode
+    :rtype: str
+    """
+    if isinstance(string, str):
+        return string
+    assert not isinstance(string, bytes)
+    return string.encode(_fs_encoding)
+
+
+_fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding()

tests/test_repositories.py

@@ -0,0 +1,62 @@
+from mock import MagicMock, patch
+from pip.index import PackageFinder
+from pip.req import InstallRequirement
+
+from piptools.repositories.pypi import PyPIRepository
+from piptools.scripts.compile import get_pip_command
+
+
+def test_pypirepo_build_dir_is_str():
+    assert isinstance(get_pypi_repository().build_dir, str)
+
+
+def test_pypirepo_source_dir_is_str():
+    assert isinstance(get_pypi_repository().source_dir, str)
+
+
+def test_pypirepo_calls_reqset_with_str_paths():
+    """
+    Make sure that paths passed to RequirementSet init are str.
+
+    Passing unicode paths on Python 2 could make pip fail later on
+    unpack, if the package contains non-ASCII file names, because
+    non-ASCII str and unicode paths cannot be combined.
+    """
+    with patch('piptools.repositories.pypi.RequirementSet') as mocked_init:
+        repo = get_pypi_repository()
+        ireq = InstallRequirement.from_line('ansible==2.4.0.0')
+
+        # Setup a mock object to be returned from the RequirementSet call
+        mocked_reqset = MagicMock()
+        mocked_init.return_value = mocked_reqset
+
+        # Do the call
+        repo.get_dependencies(ireq)
+
+        # Check that RequirementSet init is called with correct type arguments
+        assert mocked_init.call_count == 1
+        (init_call_args, init_call_kwargs) = mocked_init.call_args
+        assert isinstance(init_call_args[0], str)
+        assert isinstance(init_call_args[1], str)
+        assert isinstance(init_call_kwargs.get('download_dir'), str)
+        assert isinstance(init_call_kwargs.get('wheel_download_dir'), str)
+
+        # Check that _prepare_file is called correctly
+        assert mocked_reqset._prepare_file.call_count == 1
+        (pf_call_args, pf_call_kwargs) = mocked_reqset._prepare_file.call_args
+        (called_with_finder, called_with_ireq) = pf_call_args
+        assert isinstance(called_with_finder, PackageFinder)
+        assert called_with_ireq == ireq
+        assert not pf_call_kwargs
+
+
+def get_pypi_repository():
+    """
+    Get a PyPIRepository object for the tests.
+
+    :rtype: PyPIRepository
+    """
+    pip_command = get_pip_command()
+    pip_options = pip_command.parse_args([])[0]
+    session = pip_command._build_session(pip_options)
+    return PyPIRepository(pip_options, session)