Refactor python3's recipe into TargetPythonRecipe

Author: opacam

pythonforandroid/recipe.py

@@ -17,7 +17,8 @@
 except ImportError:
     from urllib.parse import urlparse
 from pythonforandroid.logger import (logger, info, warning, error, debug, shprint, info_main)
-from pythonforandroid.util import (urlretrieve, current_directory, ensure_dir)
+from pythonforandroid.util import (
+    urlretrieve, current_directory, ensure_dir, walk_valid_filens)
 
 # this import is necessary to keep imp.load_source from complaining :)
 if PY2:
@@ -1037,17 +1038,147 @@ def get_recipe_env(self, arch, with_flags_in_cc=True):
 
 
 class TargetPythonRecipe(Recipe):
-    '''Class for target python recipes. Sets ctx.python_recipe to point to
-    itself, so as to know later what kind of Python was built or used.'''
+    '''
+    Class for target python recipes. Sets ctx.python_recipe to point to itself,
+    so as to know later what kind of Python was built or used.
+
+    This base class is used for our main python recipes (python2 and python3)
+    which shares most of the build process.
+
+    .. note:: The recipe for python3Crystax also inherits from this base class,
+        but overwrites most of the code used here because has some different
+        build process.
+
+    .. versionchanged:: 0.6.0
+
+        - Adds the python's build process, which has been taken from the
+          python3's recipe by `inclement` and refactored here, split
+          into several methods: :meth:`build_arch` and :meth:`get_recipe_env`.
+        - Adds the attribute :attr:`configure_args`, which has been moved from
+          the original python3's recipe method :meth:`build_arch` into a static
+          class variable.
+        - Changes the return value for methods: :meth:`include_root` and
+          :meth:`link_root`.
+        - Adds some static class variables used to create the python bundle and
+          modifies the method :meth:`create_python_bundle`, refactored from the
+          python3's recipe by inclement. The added variables, also taken from
+          the mentioned recipe, are: :attr:`stdlib_dir_blacklist`,
+          :attr:`stdlib_filen_blacklist`, :attr:`site_packages_dir_blacklist`
+          and :attr:`site_packages_filen_blacklist`.
+    '''
 
     from_crystax = False
     '''True if the python is used from CrystaX, False otherwise (i.e. if
     it is built by p4a).'''
 
+    configure_args = ()
+    '''The configure arguments needed to build the python recipe. Those are
+    used in method :meth:`build_arch` (if not overwritten like python3crystax's
+    recipe does).
+    
+    .. note:: This variable should be properly set in subclass.
+    '''
+
+    stdlib_dir_blacklist = {
+        '__pycache__',
+        'test',
+        'tests',
+        'lib2to3',
+        'ensurepip',
+        'idlelib',
+        'tkinter',
+    }
+    '''The directories that we want to omit for our python bundle'''
+
+    stdlib_filen_blacklist = [
+        '*.pyc',
+        '*.exe',
+        '*.whl',
+    ]
+    '''The file extensions that we want to blacklist for our python bundle'''
+
+    site_packages_dir_blacklist = {
+        '__pycache__',
+        'tests'
+    }
+    '''The directories from site packages dir that we don't want to be included
+    in our python bundle.'''
+
+    site_packages_filen_blacklist = []
+    '''The file extensions from site packages dir that we don't want to be
+    included in our python bundle.'''
+
     def __init__(self, *args, **kwargs):
         self._ctx = None
         super(TargetPythonRecipe, self).__init__(*args, **kwargs)
 
+    def get_recipe_env(self, arch=None, with_flags_in_cc=True):
+        if self.from_crystax:
+            return \
+                super(TargetPythonRecipe, self).get_recipe_env(
+                    arch=arch, with_flags_in_cc=with_flags_in_cc)
+
+        env = environ.copy()
+        platform_name = 'android-{}'.format(self.ctx.ndk_api)
+
+        # TODO: Get this information from p4a's arch system
+        android_host = env['HOSTARCH'] = 'arm-linux-androideabi'
+
+        toolchain = '{android_host}-4.9'.format(android_host=android_host)
+        toolchain = join(self.ctx.ndk_dir, 'toolchains',
+                         toolchain, 'prebuilt', 'linux-x86_64')
+
+        env['CC'] = \
+            '{clang} -target {target} -gcc-toolchain {toolchain}'.format(
+                clang=join(self.ctx.ndk_dir, 'toolchains', 'llvm', 'prebuilt',
+                           'linux-x86_64', 'bin', 'clang'),
+                target='armv7-none-linux-androideabi',
+                toolchain=toolchain)
+        env['AR'] = join(toolchain, 'bin', android_host) + '-ar'
+        env['LD'] = join(toolchain, 'bin', android_host) + '-ld'
+        env['RANLIB'] = join(toolchain, 'bin', android_host) + '-ranlib'
+        env['READELF'] = join(toolchain, 'bin', android_host) + '-readelf'
+        env['STRIP'] = \
+            join(toolchain, 'bin', android_host) + \
+            '-strip --strip-debug --strip-unneeded'
+
+        env['PATH'] = \
+            '{hostpython_dir}:{old_path}'.format(
+                hostpython_dir=self.get_recipe(
+                    'host' + self.name, self.ctx).get_path_to_python(),
+                old_path=env['PATH'])
+
+        ndk_flags = (
+            '--sysroot={ndk_sysroot} -D__ANDROID_API__={android_api} '
+            '-isystem {ndk_android_host}').format(
+                ndk_sysroot=join(self.ctx.ndk_dir, 'sysroot'),
+                android_api=self.ctx.ndk_api,
+                ndk_android_host=join(
+                    self.ctx.ndk_dir, 'sysroot', 'usr', 'include',
+                    android_host))
+        sysroot = join(self.ctx.ndk_dir, 'platforms',
+                       platform_name, 'arch-arm')
+        env['CFLAGS'] = env.get('CFLAGS', '') + ' ' + ndk_flags
+        env['CPPFLAGS'] = env.get('CPPFLAGS', '') + ' ' + ndk_flags
+        env['LDFLAGS'] = env.get('LDFLAGS', '') + ' --sysroot={} -L{}'.format(
+            sysroot, join(sysroot, 'usr', 'lib'))
+
+        # Manually add the libs directory, and copy some object
+        # files to the current directory otherwise they aren't
+        # picked up. This seems necessary because the --sysroot
+        # setting in LDFLAGS is overridden by the other flags.
+        # TODO: Work out why this doesn't happen in the original
+        # bpo-30386 Makefile system.
+        logger.warning('Doing some hacky stuff to link properly')
+        lib_dir = join(sysroot, 'usr', 'lib')
+        env['LDFLAGS'] += ' -L{}'.format(lib_dir)
+        shprint(sh.cp, join(lib_dir, 'crtbegin_so.o'), './')
+        shprint(sh.cp, join(lib_dir, 'crtend_so.o'), './')
+
+        env['SYSROOT'] = sysroot
+
+        return env
+
     def prebuild_arch(self, arch):
         super(TargetPythonRecipe, self).prebuild_arch(arch)
         if self.from_crystax and self.ctx.ndk != 'crystax':
@@ -1056,12 +1187,14 @@ def prebuild_arch(self, arch):
             exit(1)
         self.ctx.python_recipe = self
 
-    def include_root(self, arch):
+    def include_root(self, arch_name):
         '''The root directory from which to include headers.'''
-        raise NotImplementedError('Not implemented in TargetPythonRecipe')
+        return join(self.get_build_dir(arch_name),
+                    'Include')
 
-    def link_root(self):
-        raise NotImplementedError('Not implemented in TargetPythonRecipe')
+    def link_root(self, arch_name):
+        return join(self.get_build_dir(arch_name),
+                    'android-build')
 
     @property
     def major_minor_version_string(self):
@@ -1074,7 +1207,55 @@ def create_python_bundle(self, dirn, arch):
         copying all the modules and standard library to the right
         place.
         """
-        raise NotImplementedError('{} does not implement create_python_bundle'.format(self))
+        # Bundle compiled python modules to a folder
+        modules_dir = join(dirn, 'modules')
+        ensure_dir(modules_dir)
+        # Todo: find a better way to find the build libs folder
+        modules_build_dir = join(
+            self.get_build_dir(arch.arch),
+            'android-build',
+            'build',
+            'lib.linux{}-arm-{}'.format(
+                '2' if self.version[0] == '2' else '',
+                self.major_minor_version_string
+            ))
+        module_filens = (glob.glob(join(modules_build_dir, '*.so')) +
+                         glob.glob(join(modules_build_dir, '*.py')))
+        for filen in module_filens:
+            shprint(sh.cp, filen, modules_dir)
+
+        # zip up the standard library
+        stdlib_zip = join(dirn, 'stdlib.zip')
+        with current_directory(join(self.get_build_dir(arch.arch), 'Lib')):
+            stdlib_filens = walk_valid_filens(
+                '.', self.stdlib_dir_blacklist, self.stdlib_filen_blacklist)
+            shprint(sh.zip, stdlib_zip, *stdlib_filens)
+
+        # copy the site-packages into place
+        ensure_dir(join(dirn, 'site-packages'))
+        # TODO: Improve the API around walking and copying the files
+        with current_directory(self.ctx.get_python_install_dir()):
+            filens = list(walk_valid_filens(
+                '.', self.site_packages_dir_blacklist,
+                self.site_packages_filen_blacklist))
+            for filen in filens:
+                ensure_dir(join(dirn, 'site-packages', dirname(filen)))
+                sh.cp(filen, join(dirn, 'site-packages', filen))
+
+        # copy the python .so files into place
+        python_build_dir = join(self.get_build_dir(arch.arch),
+                                'android-build')
+        python_lib_name = 'libpython' + self.major_minor_version_string
+        if self.major_minor_version_string[0] == '3':
+            python_lib_name += 'm'
+        for lib in [python_lib_name + '.so', python_lib_name + '.so.1.0']:
+            shprint(sh.cp, join(python_build_dir, lib),
+                    'libs/{}'.format(arch.arch))
+
+        info('Renaming .so files to reflect cross-compile')
+        self.reduce_object_file_names(join(dirn, 'site-packages'))
+
+        return join(dirn, 'site-packages')
 
     def reduce_object_file_names(self, dirn):
         """Recursively renames all files named XXX.cpython-...-linux-gnu.so"
@@ -1090,6 +1271,41 @@ def reduce_object_file_names(self, dirn):
                 continue
             shprint(sh.mv, filen, join(file_dirname, parts[0] + '.so'))
 
+    def build_arch(self, arch):
+        recipe_build_dir = self.get_build_dir(arch.arch)
+
+        # Create a subdirectory to actually perform the build
+        build_dir = join(recipe_build_dir, 'android-build')
+        ensure_dir(build_dir)
+
+        # TODO: Get these dynamically, like bpo-30386 does
+        sys_prefix = '/usr/local'
+        sys_exec_prefix = '/usr/local'
+
+        with current_directory(build_dir):
+            env = self.get_recipe_env(arch)
+
+            android_build = sh.Command(
+                join(recipe_build_dir,
+                     'config.guess'))().stdout.strip().decode('utf-8')
+
+            if not exists('config.status'):
+                shprint(
+                    sh.Command(join(recipe_build_dir, 'configure')),
+                    *(' '.join(self.configure_args).format(
+                                    android_host=env['HOSTARCH'],
+                                    android_build=android_build,
+                                    prefix=sys_prefix,
+                                    exec_prefix=sys_exec_prefix)).split(' '),
+                    _env=env)
+
+            if not exists('python'):
+                shprint(sh.make, 'all', _env=env)
+
+            # TODO: Look into passing the path to pyconfig.h in a
+            # better way, although this is probably acceptable
+            sh.cp('pyconfig.h', join(recipe_build_dir, 'Include'))
+
 
 class TargetHostPythonRecipe(Recipe):
     '''