[python] Fix build for Mac OSX 🍎

opacam · opacam · commit cf7f958a2f07 · 2019-08-16T15:59:50.000+02:00
It turns out that the generated binary for Mac OSX and Cygwin is not `python`...it's `python.exe` because the FS used in Mac OS (HFS+) is case insensitive per default and that would create conflicts with the generated folder `Python` by the python's build system. So we: - add cls attribute `HostPythonRecipe.python_bin` (which will be different depending on the build platform) - implement `HostPythonRecipe.should_build` to refactor a little - check the library instead of the executable to decide if we must build python's recipe (to avoid conflicts with case insensitive FS) - add message which prints the files/folders of the hostpython's build directory - fix `PythonRecipe.real_hostpython_location` because the hostpython's executable name will change depending on the build platform - fix python's build when host OS is an Mac OSX (via patch: `fix-interpreter-for-darwin.patch`) See also: https://github.com/python/cpython/blob/3.7/README.rst#build-instructions Note: @TheSin- , thanks for your debugging sessions and lines of code
diff --git a/pythonforandroid/python.py b/pythonforandroid/python.py
@@ -3,7 +3,7 @@
 build our python3 and python2 recipes and his corresponding hostpython recipes.
 '''
 
-from os.path import dirname, exists, join
+from os.path import dirname, exists, join, isfile
 from multiprocessing import cpu_count
 from shutil import copy2
 from os import environ
@@ -16,6 +16,7 @@
 from pythonforandroid.util import (
     current_directory, ensure_dir, walk_valid_filens,
     BuildInterruptingException, build_platform)
+from pythonforandroid.patching import is_darwin
 
 
 class GuestPythonRecipe(TargetPythonRecipe):
@@ -243,13 +244,20 @@ def build_arch(self, arch):
                                     exec_prefix=sys_exec_prefix)).split(' '),
                     _env=env)
 
-            if not exists('python'):
-                py_version = self.major_minor_version_string
-                if self.major_minor_version_string[0] == '3':
-                    py_version += 'm'
-                shprint(sh.make, 'all', '-j', str(cpu_count()),
-                        'INSTSONAME=libpython{version}.so'.format(
-                            version=py_version), _env=env)
+            # Some platforms use case insensitive FS (like MacOs with HFS+), so
+            # we check if we have the python's library instead of the
+            # executable, which shouldn't conflict with any existing folder in
+            # python's build dir
+            py_version = self.major_minor_version_string
+            if self.major_minor_version_string[0] == '3':
+                py_version += 'm'
+            lib_name = 'libpython{version}.so'.format(version=py_version)
+            if not isfile(join(build_dir, lib_name)):
+                shprint(
+                    sh.make, 'all', '-j', str(cpu_count()),
+                    'INSTSONAME={lib_name}'.format(lib_name=lib_name),
+                    _env=env
+                )
 
             # TODO: Look into passing the path to pyconfig.h in a
             # better way, although this is probably acceptable
@@ -382,6 +390,32 @@ class HostPythonRecipe(Recipe):
     '''The default url to download our host python recipe. This url will
     change depending on the python version set in attribute :attr:`version`.'''
 
+    @property
+    def python_bin(self):
+        '''
+        Returns the name of the python executable depending on the build
+        platform since, as explained in python docs, some platforms generates a
+        different executable name to avoid conflicts with case insensitive File
+        systems (like Mac Os X).
+
+        .. seealso::
+            https://github.com/python/cpython/blob/3.7/README.rst#build-instructions
+
+        '''
+        if is_darwin():
+            return 'python.exe'
+        return 'python'
+
+    def should_build(self, arch):
+        python_bin = join(
+            self.get_build_dir(arch.arch), self.build_subdir, self.python_bin
+        )
+        if exists(python_bin):
+            # no need to build, but we must set:
+            self.ctx.hostpython = python_bin
+            return False
+        return True
+
     def get_build_container_dir(self, arch=None):
         choices = self.check_recipe_choices()
         dir_name = '-'.join([self.name] + choices)
@@ -404,22 +438,25 @@ def build_arch(self, arch):
         build_dir = join(recipe_build_dir, self.build_subdir)
         ensure_dir(build_dir)
 
-        if not exists(join(build_dir, 'python')):
-            with current_directory(recipe_build_dir):
-                # Configure the build
-                with current_directory(build_dir):
-                    if not exists('config.status'):
-                        shprint(
-                            sh.Command(join(recipe_build_dir, 'configure')))
+        with current_directory(recipe_build_dir):
+            # Configure the build
+            with current_directory(build_dir):
+                if not exists('config.status'):
+                    shprint(
+                        sh.Command(join(recipe_build_dir, 'configure')))
 
-                # Create the Setup file. This copying from Setup.dist
-                # seems to be the normal and expected procedure.
-                shprint(sh.cp, join('Modules', 'Setup.dist'),
-                        join(build_dir, 'Modules', 'Setup'))
+            # Create the Setup file. This copying from Setup.dist
+            # seems to be the normal and expected procedure.
+            shprint(sh.cp, join('Modules', 'Setup.dist'),
+                    join(build_dir, 'Modules', 'Setup'))
 
-                shprint(sh.make, '-j', str(cpu_count()), '-C', build_dir)
-        else:
-            info('Skipping {name} ({version}) build, as it has already '
-                 'been completed'.format(name=self.name, version=self.version))
+            shprint(sh.make, '-j', str(cpu_count()), '-C', build_dir)
+
+        # Since we got different python's executable names depending on the
+        # build platform, we add a a debug message which prints the generated
+        # files/folders, in case that we need that...
+        python_dir = join(self.get_build_dir(arch.arch), self.build_subdir)
+        info("Files/folders in python's build directory are:")
+        shprint(sh.ls, python_dir)
 
-        self.ctx.hostpython = join(build_dir, 'python')
+        self.ctx.hostpython = join(build_dir, self.python_bin)
diff --git a/pythonforandroid/recipe.py b/pythonforandroid/recipe.py
@@ -773,9 +773,13 @@ def clean_build(self, arch=None):
     @property
     def real_hostpython_location(self):
         host_name = 'host{}'.format(self.ctx.python_recipe.name)
-        host_build = Recipe.get_recipe(host_name, self.ctx).get_build_dir()
         if host_name in ['hostpython2', 'hostpython3']:
-            return join(host_build, 'native-build', 'python')
+            python_recipe = Recipe.get_recipe(host_name, self.ctx)
+            return join(
+                python_recipe.get_build_dir(),
+                'native-build',
+                python_recipe.python_bin
+            )
         else:
             python_recipe = self.ctx.python_recipe
             return 'python{}'.format(python_recipe.version)
diff --git a/pythonforandroid/recipes/python2/__init__.py b/pythonforandroid/recipes/python2/__init__.py
@@ -2,6 +2,7 @@
 from pythonforandroid.recipe import Recipe
 from pythonforandroid.python import GuestPythonRecipe
 from pythonforandroid.logger import shprint, warning
+from pythonforandroid.patching import is_darwin
 import sh
 
 
@@ -33,6 +34,8 @@ class Python2Recipe(GuestPythonRecipe):
                'patches/fix-posix-declarations.patch',
                'patches/fix-pwd-gecos.patch',
                'patches/fix-ctypes-util-find-library.patch']
+    if is_darwin():
+        patches += ["patches/fix-interpreter-for-darwin.patch"]
 
     configure_args = ('--host={android_host}',
                       '--build={android_build}',
diff --git a/pythonforandroid/recipes/python2/patches/fix-interpreter-for-darwin.patch b/pythonforandroid/recipes/python2/patches/fix-interpreter-for-darwin.patch
@@ -0,0 +1,20 @@
+--- Python-2.7.15/configure.orig	2018-04-30 00:47:33.000000000 +0200
++++ Python-2.7.15/configure	2019-08-16 14:09:48.696030207 +0200
+@@ -2903,7 +2903,7 @@ case $host_os in *\ *) host_os=`echo "$h
+ # pybuilddir.txt will be created by --generate-posix-vars in the Makefile
+ rm -f pybuilddir.txt
+ 
+-for ac_prog in python$PACKAGE_VERSION python3 python
++for ac_prog in python$PACKAGE_VERSION python3 python.exe python
+ do
+   # Extract the first word of "$ac_prog", so it can be a program name with args.
+ set dummy $ac_prog; ac_word=$2
+@@ -2952,7 +2952,7 @@ if test "$cross_compiling" = yes; then
+     { $as_echo "$as_me:${as_lineno-$LINENO}: checking for python interpreter for cross build" >&5
+ $as_echo_n "checking for python interpreter for cross build... " >&6; }
+     if test -z "$PYTHON_FOR_BUILD"; then
+-        for interp in python$PACKAGE_VERSION python2 python; do
++        for interp in python$PACKAGE_VERSION python2 python.exe python; do
+ 	    which $interp >/dev/null 2>&1 || continue
+ 	    if $interp -c 'import sys;sys.exit(not (sys.version_info[:2] >= (2,7) and sys.version_info[0] < 3))'; then
+ 	        break
diff --git a/pythonforandroid/recipes/python3/__init__.py b/pythonforandroid/recipes/python3/__init__.py
@@ -1,6 +1,7 @@
 import sh
 from pythonforandroid.python import GuestPythonRecipe
 from pythonforandroid.recipe import Recipe
+from pythonforandroid.patching import is_darwin
 
 
 class Python3Recipe(GuestPythonRecipe):
@@ -23,9 +24,10 @@ class Python3Recipe(GuestPythonRecipe):
     name = 'python3'
 
     patches = ["patches/fix-ctypes-util-find-library.patch"]
-
     if sh.which('lld') is not None:
-        patches = patches + ["patches/remove-fix-cortex-a8.patch"]
+        patches += ["patches/remove-fix-cortex-a8.patch"]
+    if is_darwin():
+        patches += ["patches/fix-interpreter-for-darwin.patch"]
 
     depends = ['hostpython3', 'sqlite3', 'openssl', 'libffi']
     conflicts = ['python2']
diff --git a/pythonforandroid/recipes/python3/patches/fix-interpreter-for-darwin.patch b/pythonforandroid/recipes/python3/patches/fix-interpreter-for-darwin.patch
@@ -0,0 +1,21 @@
+diff --git a/configure b/configure
+--- a/configure	2019-08-06 02:35:17.000000000 +0000
++++ b/configure	2019-08-06 02:36:22.000000000 +0000
+@@ -2868,7 +2868,7 @@
+ # pybuilddir.txt will be created by --generate-posix-vars in the Makefile
+ rm -f pybuilddir.txt
+
+-for ac_prog in python$PACKAGE_VERSION python3 python
++for ac_prog in python$PACKAGE_VERSION python3 python.exe python
+ do
+   # Extract the first word of "$ac_prog", so it can be a program name with args.
+ set dummy $ac_prog; ac_word=$2
+@@ -2917,7 +2917,7 @@
+     { $as_echo "$as_me:${as_lineno-$LINENO}: checking for python interpreter for cross build" >&5
+ $as_echo_n "checking for python interpreter for cross build... " >&6; }
+     if test -z "$PYTHON_FOR_BUILD"; then
+-        for interp in python$PACKAGE_VERSION python3 python; do
++        for interp in python$PACKAGE_VERSION python3 python.exe python; do
+ 	    which $interp >/dev/null 2>&1 || continue
+ 	    if $interp -c "import sys;sys.exit(not '.'.join(str(n) for n in sys.version_info[:2]) == '$PACKAGE_VERSION')"; then
+ 	        break
