Merge pull request #7102 from mayeut/manylinux2014

Add manylinux2014 support

Author: chrahunt

news/7102.feature

@@ -0,0 +1,4 @@
+Implement manylinux2014 platform tag support.  manylinux2014 is the successor
+to manylinux2010.  It allows carefully compiled binary wheels to be installed
+on compatible Linux platforms.  The manylinux2014 platform tag definition can
+be found in `PEP599 <https://www.python.org/dev/peps/pep-0599/>`_.

src/pip/_internal/pep425tags.py

@@ -164,6 +164,32 @@ def get_platform():
     return result
 
 
+def is_linux_armhf():
+    # type: () -> bool
+    if get_platform() != "linux_armv7l":
+        return False
+    # hard-float ABI can be detected from the ELF header of the running
+    # process
+    try:
+        with open(sys.executable, 'rb') as f:
+            elf_header_raw = f.read(40)  # read 40 first bytes of ELF header
+    except (IOError, OSError, TypeError):
+        return False
+    if elf_header_raw is None or len(elf_header_raw) < 40:
+        return False
+    if isinstance(elf_header_raw, str):
+        elf_header = [ord(c) for c in elf_header_raw]
+    else:
+        elf_header = [b for b in elf_header_raw]
+    result = elf_header[0:4] == [0x7f, 0x45, 0x4c, 0x46]  # ELF magic number
+    result &= elf_header[4:5] == [1]  # 32-bit ELF
+    result &= elf_header[5:6] == [1]  # little-endian
+    result &= elf_header[18:20] == [0x28, 0]  # ARM machine
+    result &= elf_header[39:40] == [5]  # ARM EABIv5
+    result &= (elf_header[37:38][0] & 4) == 4  # EF_ARM_ABI_FLOAT_HARD
+    return result
+
+
 def is_manylinux1_compatible():
     # type: () -> bool
     # Only Linux, and only x86-64 / i686
@@ -200,6 +226,32 @@ def is_manylinux2010_compatible():
     return pip._internal.utils.glibc.have_compatible_glibc(2, 12)
 
 
+def is_manylinux2014_compatible():
+    # type: () -> bool
+    # Only Linux, and only supported architectures
+    platform = get_platform()
+    if platform not in {"linux_x86_64", "linux_i686", "linux_aarch64",
+                        "linux_armv7l", "linux_ppc64", "linux_ppc64le",
+                        "linux_s390x"}:
+        return False
+
+    # check for hard-float ABI in case we're running linux_armv7l not to
+    # install hard-float ABI wheel in a soft-float ABI environment
+    if platform == "linux_armv7l" and not is_linux_armhf():
+        return False
+
+    # Check for presence of _manylinux module
+    try:
+        import _manylinux
+        return bool(_manylinux.manylinux2014_compatible)
+    except (ImportError, AttributeError):
+        # Fall through to heuristic check below
+        pass
+
+    # Check glibc version. CentOS 7 uses glibc 2.17.
+    return pip._internal.utils.glibc.have_compatible_glibc(2, 17)
+
+
 def get_darwin_arches(major, minor, machine):
     # type: (int, int, str) -> List[str]
     """Return a list of supported arches (including group arches) for
@@ -333,6 +385,16 @@ def get_supported(
             else:
                 # arch pattern didn't match (?!)
                 arches = [arch]
+        elif arch_prefix == 'manylinux2014':
+            arches = [arch]
+            # manylinux1/manylinux2010 wheels run on most manylinux2014 systems
+            # with the exception of wheels depending on ncurses. PEP 599 states
+            # manylinux1/manylinux2010 wheels should be considered
+            # manylinux2014 wheels:
+            # https://www.python.org/dev/peps/pep-0599/#backwards-compatibility-with-manylinux2010-wheels
+            if arch_suffix in {'i686', 'x86_64'}:
+                arches.append('manylinux2010' + arch_sep + arch_suffix)
+                arches.append('manylinux1' + arch_sep + arch_suffix)
         elif arch_prefix == 'manylinux2010':
             # manylinux1 wheels run on most manylinux2010 systems with the
             # exception of wheels depending on ncurses. PEP 571 states
@@ -341,6 +403,8 @@ def get_supported(
             arches = [arch, 'manylinux1' + arch_sep + arch_suffix]
         elif platform is None:
             arches = []
+            if is_manylinux2014_compatible():
+                arches.append('manylinux2014' + arch_sep + arch_suffix)
             if is_manylinux2010_compatible():
                 arches.append('manylinux2010' + arch_sep + arch_suffix)
             if is_manylinux1_compatible():

tests/functional/test_download.py

@@ -331,6 +331,7 @@ class TestDownloadPlatformManylinuxes(object):
         "linux_x86_64",
         "manylinux1_x86_64",
         "manylinux2010_x86_64",
+        "manylinux2014_x86_64",
     ])
     def test_download_universal(self, platform, script, data):
         """
@@ -353,6 +354,9 @@ def test_download_universal(self, platform, script, data):
         ("manylinux1_x86_64", "manylinux1_x86_64"),
         ("manylinux1_x86_64", "manylinux2010_x86_64"),
         ("manylinux2010_x86_64", "manylinux2010_x86_64"),
+        ("manylinux1_x86_64", "manylinux2014_x86_64"),
+        ("manylinux2010_x86_64", "manylinux2014_x86_64"),
+        ("manylinux2014_x86_64", "manylinux2014_x86_64"),
     ])
     def test_download_compatible_manylinuxes(
             self, wheel_abi, platform, script, data,

tests/unit/test_pep425tags.py

@@ -137,6 +137,7 @@ def test_manual_abi_dm_flags(self):
 @pytest.mark.parametrize('is_manylinux_compatible', [
     pep425tags.is_manylinux1_compatible,
     pep425tags.is_manylinux2010_compatible,
+    pep425tags.is_manylinux2014_compatible,
 ])
 class TestManylinuxTags(object):
     """
@@ -156,28 +157,28 @@ def test_manylinux_compatible_on_linux_x86_64(self,
     @patch('pip._internal.pep425tags.get_platform', lambda: 'linux_i686')
     @patch('pip._internal.utils.glibc.have_compatible_glibc',
            lambda major, minor: True)
-    def test_manylinux1_compatible_on_linux_i686(self,
-                                                 is_manylinux_compatible):
+    def test_manylinux_compatible_on_linux_i686(self,
+                                                is_manylinux_compatible):
         """
-        Test that manylinux1 is enabled on linux_i686
+        Test that manylinuxes are enabled on linux_i686
         """
         assert is_manylinux_compatible()
 
     @patch('pip._internal.pep425tags.get_platform', lambda: 'linux_x86_64')
     @patch('pip._internal.utils.glibc.have_compatible_glibc',
            lambda major, minor: False)
-    def test_manylinux1_2(self, is_manylinux_compatible):
+    def test_manylinux_2(self, is_manylinux_compatible):
         """
-        Test that manylinux1 is disabled with incompatible glibc
+        Test that manylinuxes are disabled with incompatible glibc
         """
         assert not is_manylinux_compatible()
 
     @patch('pip._internal.pep425tags.get_platform', lambda: 'arm6vl')
     @patch('pip._internal.utils.glibc.have_compatible_glibc',
            lambda major, minor: True)
-    def test_manylinux1_3(self, is_manylinux_compatible):
+    def test_manylinux_3(self, is_manylinux_compatible):
         """
-        Test that manylinux1 is disabled on arm6vl
+        Test that manylinuxes are disabled on arm6vl
         """
         assert not is_manylinux_compatible()
 
@@ -186,6 +187,8 @@ class TestManylinux1Tags(object):
 
     @patch('pip._internal.pep425tags.is_manylinux2010_compatible',
            lambda: False)
+    @patch('pip._internal.pep425tags.is_manylinux2014_compatible',
+           lambda: False)
     @patch('pip._internal.pep425tags.get_platform', lambda: 'linux_x86_64')
     @patch('pip._internal.utils.glibc.have_compatible_glibc',
            lambda major, minor: True)
@@ -210,6 +213,8 @@ def test_manylinux1_tag_is_first(self):
 
 class TestManylinux2010Tags(object):
 
+    @patch('pip._internal.pep425tags.is_manylinux2014_compatible',
+           lambda: False)
     @patch('pip._internal.pep425tags.get_platform', lambda: 'linux_x86_64')
     @patch('pip._internal.utils.glibc.have_compatible_glibc',
            lambda major, minor: True)
@@ -253,3 +258,55 @@ def test_manylinux2010_implies_manylinux1(self, manylinux2010, manylinux1):
             if arches == ['any']:
                 continue
             assert arches[:2] == [manylinux2010, manylinux1]
+
+
+class TestManylinux2014Tags(object):
+
+    @patch('pip._internal.pep425tags.get_platform', lambda: 'linux_x86_64')
+    @patch('pip._internal.utils.glibc.have_compatible_glibc',
+           lambda major, minor: True)
+    @patch('sys.platform', 'linux2')
+    def test_manylinux2014_tag_is_first(self):
+        """
+        Test that the more specific tag manylinux2014 comes first.
+        """
+        groups = {}
+        for pyimpl, abi, arch in pep425tags.get_supported():
+            groups.setdefault((pyimpl, abi), []).append(arch)
+
+        for arches in groups.values():
+            if arches == ['any']:
+                continue
+            # Expect the most specific arch first:
+            if len(arches) == 5:
+                assert arches == ['manylinux2014_x86_64',
+                                  'manylinux2010_x86_64',
+                                  'manylinux1_x86_64',
+                                  'linux_x86_64',
+                                  'any']
+            else:
+                assert arches == ['manylinux2014_x86_64',
+                                  'manylinux2010_x86_64',
+                                  'manylinux1_x86_64',
+                                  'linux_x86_64']
+
+    @pytest.mark.parametrize("manylinuxA,manylinuxB", [
+        ("manylinux2014_x86_64", ["manylinux2010_x86_64",
+                                  "manylinux1_x86_64"]),
+        ("manylinux2014_i686", ["manylinux2010_i686", "manylinux1_i686"]),
+    ])
+    def test_manylinuxA_implies_manylinuxB(self, manylinuxA, manylinuxB):
+        """
+        Specifying manylinux2014 implies manylinux2010/manylinux1.
+        """
+        groups = {}
+        supported = pep425tags.get_supported(platform=manylinuxA)
+        for pyimpl, abi, arch in supported:
+            groups.setdefault((pyimpl, abi), []).append(arch)
+
+        expected_arches = [manylinuxA]
+        expected_arches.extend(manylinuxB)
+        for arches in groups.values():
+            if arches == ['any']:
+                continue
+            assert arches[:3] == expected_arches