[python] Fix build for case-insensitive FS 🍎

It turns out that the generated python binary for some builds are named `python.exe` instead of `python`. This depends on the File System where the build happens. It will be named `python.exe` when the FS is case-insensitive (Mac OSX and Cygwin), but it will be named `python` when the FS is case-sensitive (most GNU Linux distributions). The proposed solution consists in make a copy of the generated python binary with a given name (python3 or python2) so this way

To achieve this goal we refactor a little our `HostPythonRecipe`:
  - add private property `HostPythonRecipe._exe_name` (the name of the python executable based on major version)
  - add public property `HostPythonRecipe.python_exe` (the full path of the python executable)
  - implement `HostPythonRecipe.should_build`

And also it's affected the `GuestPythonRecipe`, because we need to use the generated python executable by `HostPythonRecipe`:
  - add private property `GuestPythonRecipe._libpython` (python's library name with extension...hardcoded for now...)
  - implement `GuestPythonRecipe.should_build`... to check the library instead of the executable so we avoid conflicts with case-insensitive FS

We also need:
  - fix `PythonRecipe.real_hostpython_location` because the name of our host python executable will depend on major version
  - fix python2 interpreter (fix-interpreter-version.patch)

Note: the variation of the name of the python's executable is mentioned at python's build docs (https://github.com/python/cpython/blob/3.7/README.rst#build-instructions)

Note: @TheSin- , ¡¡¡thanks for your debugging sessions!!!

Author: opacam

pythonforandroid/python.py

@@ -3,7 +3,7 @@
 build our python3 and python2 recipes and his corresponding hostpython recipes.
 '''
 
-from os.path import dirname, exists, join
+from os.path import dirname, exists, join, isfile
 from multiprocessing import cpu_count
 from shutil import copy2
 from os import environ
@@ -205,6 +205,17 @@ def add_flags(include_flags, link_dirs, link_libs):
                       recipe.link_dirs_flags(arch), recipe.link_libs_flags())
         return env
 
+    @property
+    def _libpython(self):
+        '''return the python's library name (with extension)'''
+        py_version = self.major_minor_version_string
+        if self.major_minor_version_string[0] == '3':
+            py_version += 'm'
+        return 'libpython{version}.so'.format(version=py_version)
+
+    def should_build(self, arch):
+        return not isfile(join(self.link_root(arch.arch), self._libpython))
+
     def prebuild_arch(self, arch):
         super(TargetPythonRecipe, self).prebuild_arch(arch)
         self.ctx.python_recipe = self
@@ -243,13 +254,11 @@ def build_arch(self, arch):
                                     exec_prefix=sys_exec_prefix)).split(' '),
                     _env=env)
 
-            if not exists('python'):
-                py_version = self.major_minor_version_string
-                if self.major_minor_version_string[0] == '3':
-                    py_version += 'm'
-                shprint(sh.make, 'all', '-j', str(cpu_count()),
-                        'INSTSONAME=libpython{version}.so'.format(
-                            version=py_version), _env=env)
+            shprint(
+                sh.make, 'all', '-j', str(cpu_count()),
+                'INSTSONAME={lib_name}'.format(lib_name=self._libpython),
+                _env=env
+            )
 
             # TODO: Look into passing the path to pyconfig.h in a
             # better way, although this is probably acceptable
@@ -382,6 +391,30 @@ class HostPythonRecipe(Recipe):
     '''The default url to download our host python recipe. This url will
     change depending on the python version set in attribute :attr:`version`.'''
 
+    @property
+    def _exe_name(self):
+        '''
+        Returns the name of the python executable depending on the version.
+        '''
+        if not self.version:
+            raise BuildInterruptingException(
+                'The hostpython recipe must have set version'
+            )
+        version = self.version.split('.')[0]
+        return 'python{major_version}'.format(major_version=version)
+
+    @property
+    def python_exe(self):
+        '''Returns the full path of the hostpython executable.'''
+        return join(self.get_path_to_python(), self._exe_name)
+
+    def should_build(self, arch):
+        if exists(self.python_exe):
+            # no need to build, but we must set hostpython for our Context
+            self.ctx.hostpython = self.python_exe
+            return False
+        return True
+
     def get_build_container_dir(self, arch=None):
         choices = self.check_recipe_choices()
         dir_name = '-'.join([self.name] + choices)
@@ -404,22 +437,28 @@ def build_arch(self, arch):
         build_dir = join(recipe_build_dir, self.build_subdir)
         ensure_dir(build_dir)
 
-        if not exists(join(build_dir, 'python')):
-            with current_directory(recipe_build_dir):
-                # Configure the build
-                with current_directory(build_dir):
-                    if not exists('config.status'):
-                        shprint(
-                            sh.Command(join(recipe_build_dir, 'configure')))
-
-                # Create the Setup file. This copying from Setup.dist
-                # seems to be the normal and expected procedure.
-                shprint(sh.cp, join('Modules', 'Setup.dist'),
-                        join(build_dir, 'Modules', 'Setup'))
-
-                shprint(sh.make, '-j', str(cpu_count()), '-C', build_dir)
-        else:
-            info('Skipping {name} ({version}) build, as it has already '
-                 'been completed'.format(name=self.name, version=self.version))
-
-        self.ctx.hostpython = join(build_dir, 'python')
+        with current_directory(recipe_build_dir):
+            # Configure the build
+            with current_directory(build_dir):
+                if not exists('config.status'):
+                    shprint(sh.Command(join(recipe_build_dir, 'configure')))
+
+            # Create the Setup file. This copying from Setup.dist
+            # seems to be the normal and expected procedure.
+            shprint(sh.cp, join('Modules', 'Setup.dist'),
+                    join(build_dir, 'Modules', 'Setup'))
+
+            shprint(sh.make, '-j', str(cpu_count()), '-C', build_dir)
+
+            # make a copy of the python executable giving it the name we want,
+            # because we got different python's executable names depending on
+            # the fs being case-insensitive (Mac OS X, Cygwin...) or
+            # case-sensitive (linux)...so this way we will have an unique name
+            # for our hostpython, regarding the used fs
+            for exe_name in ['python.exe', 'python']:
+                exe = join(self.get_path_to_python(), exe_name)
+                if isfile(exe):
+                    shprint(sh.cp, exe, self.python_exe)
+                    break
+
+        self.ctx.hostpython = self.python_exe

pythonforandroid/recipe.py

@@ -773,9 +773,9 @@ def clean_build(self, arch=None):
     @property
     def real_hostpython_location(self):
         host_name = 'host{}'.format(self.ctx.python_recipe.name)
-        host_build = Recipe.get_recipe(host_name, self.ctx).get_build_dir()
         if host_name in ['hostpython2', 'hostpython3']:
-            return join(host_build, 'native-build', 'python')
+            python_recipe = Recipe.get_recipe(host_name, self.ctx)
+            return python_recipe.python_exe
         else:
             python_recipe = self.ctx.python_recipe
             return 'python{}'.format(python_recipe.version)

pythonforandroid/recipes/python2/__init__.py

@@ -32,7 +32,9 @@ class Python2Recipe(GuestPythonRecipe):
                'patches/fix-gethostbyaddr.patch',
                'patches/fix-posix-declarations.patch',
                'patches/fix-pwd-gecos.patch',
-               'patches/fix-ctypes-util-find-library.patch']
+               'patches/fix-ctypes-util-find-library.patch',
+               'patches/fix-interpreter-version.patch',
+    ]
 
     configure_args = ('--host={android_host}',
                       '--build={android_build}',

pythonforandroid/recipes/python2/patches/fix-interpreter-version.patch

@@ -0,0 +1,11 @@
+--- Python-2.7.15/configure.orig	2018-04-30 00:47:33.000000000 +0200
++++ Python-2.7.15/configure	2019-08-16 14:09:48.696030207 +0200
+@@ -2903,7 +2903,7 @@ case $host_os in *\ *) host_os=`echo "$h
+ # pybuilddir.txt will be created by --generate-posix-vars in the Makefile
+ rm -f pybuilddir.txt
+ 
+-for ac_prog in python$PACKAGE_VERSION python3 python
++for ac_prog in python$PACKAGE_VERSION python2 python
+ do
+   # Extract the first word of "$ac_prog", so it can be a program name with args.
+ set dummy $ac_prog; ac_word=$2