Merge pull request #1412 from inclement/python3_recipe

Python 3 recipe

Author: inclement

.travis.yml

@@ -19,12 +19,11 @@ env:
     - ANDROID_NDK_HOME=/opt/android/android-ndk
     - CRYSTAX_NDK_HOME=/opt/android/crystax-ndk
   matrix:
-    - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python2.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME'
     # overrides requirements to skip `peewee` pure python module, see:
     # https://github.com/kivy/python-for-android/issues/1263#issuecomment-390421054
     - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python2_sqlite_openssl.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME --requirements sdl2,pyjnius,kivy,python2,openssl,requests,sqlite3,setuptools'
     - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python2.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME --bootstrap sdl2 --requirements python2,numpy'
-    - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python3.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $CRYSTAX_NDK_HOME --requirements python3crystax,setuptools,android,sdl2,pyjnius,kivy'
+    - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python3crystax.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $CRYSTAX_NDK_HOME --requirements python3crystax,setuptools,android,sdl2,pyjnius,kivy'
     # builds only the recipes that moved
     - COMMAND='. venv/bin/activate && ./ci/rebuild_updated_recipes.py'
 

Dockerfile

@@ -84,6 +84,7 @@ RUN mkdir --parents "${ANDROID_SDK_HOME}/.android/" && \
 	echo '### User Sources for Android SDK Manager' > "${ANDROID_SDK_HOME}/.android/repositories.cfg"
 RUN yes | "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" --licenses
 RUN "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "platforms;android-19" && \
+    "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "platforms;android-27" && \
     "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "build-tools;26.0.2" && \
     chmod +x "${ANDROID_SDK_HOME}/tools/bin/avdmanager"
 

doc/source/buildoptions.rst

@@ -17,16 +17,23 @@ This option builds Python 2.7.2 for your selected Android
 architecture. There are no special requirements, all the building is
 done locally.
 
-The python2 build is also the way python-for-android originally
-worked, even in the old toolchain.
-
 
 python3
 ~~~~~~~
 
-.. warning::
-   Python3 support is experimental, and some of these details
-   may change as it is improved and fully stabilised.
+Python3 is supported in two ways. The default method uses CPython 3.7+
+and works with any recent version of the Android NDK.
+
+Select Python 3 by adding it to your requirements,
+e.g. ``--requirements=python3``.
+
+
+CrystaX python3
+###############
+
+.. warning:: python-for-android originally supported Python 3 using the CrystaX
+             NDK. This support is now being phased out as CrystaX is no longer
+             actively developed.
 
 .. note:: You must manually download the `CrystaX NDK
    <https://www.crystax.net/android/ndk>`__ and tell
@@ -42,11 +49,6 @@ Google's official NDK which includes many improvements. You
 python3. You can get it `here
 <https://www.crystax.net/en/download>`__.
 
-The python3crystax build is handled quite differently to python2 so
-there may be bugs or surprising behaviours. If you come across any,
-feel free to `open an issue
-<https://github.com/kivy/python-for-android>`__.
-
 .. _bootstrap_build_options:
 
 Bootstrap options

doc/source/quickstart.rst

@@ -114,15 +114,17 @@ Then, you can edit your ``~/.bashrc`` or other favorite shell to include new env
     # Adjust the paths!
     export ANDROIDSDK="$HOME/Documents/android-sdk-21"
     export ANDROIDNDK="$HOME/Documents/android-ndk-r10e"
-    export ANDROIDAPI="19"  # Target API version of your application
+    export ANDROIDAPI="26"  # Target API version of your application
+    export NDKAPI="19"  # Minimum supported API version of your application
     export ANDROIDNDKVER="r10e"  # Version of the NDK you installed
 
 You have the possibility to configure on any command the PATH to the SDK, NDK and Android API using:
 
-- :code:`--sdk_dir PATH` as an equivalent of `$ANDROIDSDK`
-- :code:`--ndk_dir PATH` as an equivalent of `$ANDROIDNDK`
-- :code:`--android_api VERSION` as an equivalent of `$ANDROIDAPI`
-- :code:`--ndk_version VERSION` as an equivalent of `$ANDROIDNDKVER`
+- :code:`--sdk-dir PATH` as an equivalent of `$ANDROIDSDK`
+- :code:`--ndk-dir PATH` as an equivalent of `$ANDROIDNDK`
+- :code:`--android-api VERSION` as an equivalent of `$ANDROIDAPI`
+- :code:`--ndk-api VERSION` as an equivalent of `$NDKAPI`
+- :code:`--ndk-version VERSION` as an equivalent of `$ANDROIDNDKVER`
 
 
 Usage

pythonforandroid/archs.py

@@ -35,7 +35,7 @@ def get_env(self, with_flags_in_cc=True):
 
         env['CFLAGS'] = ' '.join([
             '-DANDROID', '-mandroid', '-fomit-frame-pointer'
-            ' -D__ANDROID_API__={}'.format(self.ctx._android_api),
+            ' -D__ANDROID_API__={}'.format(self.ctx.ndk_api),
             ])
         env['LDFLAGS'] = ' '
 
@@ -133,6 +133,7 @@ def get_env(self, with_flags_in_cc=True):
         env['PATH'] = environ['PATH']
 
         env['ARCH'] = self.arch
+        env['NDK_API'] = 'android-{}'.format(str(self.ctx.ndk_api))
 
         if self.ctx.python_recipe and self.ctx.python_recipe.from_crystax:
             env['CRYSTAX_PYTHON_VERSION'] = self.ctx.python_recipe.version

pythonforandroid/bootstrap.py

@@ -2,7 +2,6 @@
 from os import listdir
 import sh
 import glob
-import json
 import importlib
 
 from pythonforandroid.logger import (warning, shprint, info, logger,
@@ -102,21 +101,10 @@ def prepare_build_dir(self):
                 fileh.write('target=android-{}'.format(self.ctx.android_api))
 
     def prepare_dist_dir(self, name):
-        # self.dist_dir = self.get_dist_dir(name)
         ensure_dir(self.dist_dir)
 
     def run_distribute(self):
-        # print('Default bootstrap being used doesn\'t know how '
-        #       'to distribute...failing.')
-        # exit(1)
-        with current_directory(self.dist_dir):
-            info('Saving distribution info')
-            with open('dist_info.json', 'w') as fileh:
-                json.dump({'dist_name': self.ctx.dist_name,
-                           'bootstrap': self.ctx.bootstrap.name,
-                           'archs': [arch.arch for arch in self.ctx.archs],
-                           'recipes': self.ctx.recipe_build_order + self.ctx.python_modules},
-                          fileh)
+        self.distribution.save_info(self.dist_dir)
 
     @classmethod
     def list_bootstraps(cls):
@@ -254,9 +242,15 @@ def strip_libraries(self, arch):
             warning('Can\'t find strip in PATH...')
             return
         strip = sh.Command(strip)
-        filens = shprint(sh.find, join(self.dist_dir, 'private'),
-                         join(self.dist_dir, 'libs'),
-                         '-iname', '*.so', _env=env).stdout.decode('utf-8')
+
+        if self.ctx.python_recipe.name == 'python2':
+            filens = shprint(sh.find, join(self.dist_dir, 'private'),
+                             join(self.dist_dir, 'libs'),
+                             '-iname', '*.so', _env=env).stdout.decode('utf-8')
+        else:
+            filens = shprint(sh.find, join(self.dist_dir, '_python_bundle', '_python_bundle', 'modules'),
+                             join(self.dist_dir, 'libs'),
+                             '-iname', '*.so', _env=env).stdout.decode('utf-8')
         logger.info('Stripping libraries in private dir')
         for filen in filens.split('\n'):
             try:

pythonforandroid/bootstraps/sdl2/__init__.py

@@ -1,27 +1,21 @@
 from pythonforandroid.toolchain import (
     Bootstrap, shprint, current_directory, info, info_main)
 from pythonforandroid.util import ensure_dir
-from os.path import join, exists, curdir, abspath
-from os import walk
-import glob
+from os.path import join, exists
 import sh
 
 
-EXCLUDE_EXTS = (".py", ".pyc", ".so.o", ".so.a", ".so.libs", ".pyx")
-
-
 class SDL2GradleBootstrap(Bootstrap):
-    name = 'sdl2_gradle'
+    name = 'sdl2'
 
-    recipe_depends = ['sdl2', ('python2', 'python3crystax')]
+    recipe_depends = ['sdl2', ('python2', 'python3', 'python3crystax')]
 
     def run_distribute(self):
         info_main("# Creating Android project ({})".format(self.name))
 
         arch = self.ctx.archs[0]
         python_install_dir = self.ctx.get_python_install_dir()
         from_crystax = self.ctx.python_recipe.from_crystax
-        crystax_python_dir = join("crystax_python", "crystax_python")
 
         if len(self.ctx.archs) > 1:
             raise ValueError("SDL2/gradle support only one arch")
@@ -30,7 +24,7 @@ def run_distribute(self):
         shprint(sh.rm, "-rf", self.dist_dir)
         shprint(sh.cp, "-r", self.build_dir, self.dist_dir)
 
-        # either the build use environemnt variable (ANDROID_HOME)
+        # either the build use environment variable (ANDROID_HOME)
         # or the local.properties if exists
         with current_directory(self.dist_dir):
             with open('local.properties', 'w') as fileh:
@@ -39,91 +33,31 @@ def run_distribute(self):
         with current_directory(self.dist_dir):
             info("Copying Python distribution")
 
-            if not exists("private") and not from_crystax:
-                ensure_dir("private")
-            if not exists("crystax_python") and from_crystax:
-                ensure_dir(crystax_python_dir)
-
             hostpython = sh.Command(self.ctx.hostpython)
-            if not from_crystax:
+            if self.ctx.python_recipe.name == 'python2':
                 try:
                     shprint(hostpython, '-OO', '-m', 'compileall',
                             python_install_dir,
                             _tail=10, _filterout="^Listing")
                 except sh.ErrorReturnCode:
                     pass
-                if not exists('python-install'):
+                if 'python2' in self.ctx.recipe_build_order and not exists('python-install'):
                     shprint(
                         sh.cp, '-a', python_install_dir, './python-install')
 
             self.distribute_libs(arch, [self.ctx.get_libs_dir(arch.arch)])
             self.distribute_javaclasses(self.ctx.javaclass_dir,
                                         dest_dir=join("src", "main", "java"))
 
-            if not from_crystax:
-                info("Filling private directory")
-                if not exists(join("private", "lib")):
-                    info("private/lib does not exist, making")
-                    shprint(sh.cp, "-a",
-                            join("python-install", "lib"), "private")
-                shprint(sh.mkdir, "-p",
-                        join("private", "include", "python2.7"))
-
-                libpymodules_fn = join("libs", arch.arch, "libpymodules.so")
-                if exists(libpymodules_fn):
-                    shprint(sh.mv, libpymodules_fn, 'private/')
-                shprint(sh.cp,
-                        join('python-install', 'include',
-                             'python2.7', 'pyconfig.h'),
-                        join('private', 'include', 'python2.7/'))
-
-                info('Removing some unwanted files')
-                shprint(sh.rm, '-f', join('private', 'lib', 'libpython2.7.so'))
-                shprint(sh.rm, '-rf', join('private', 'lib', 'pkgconfig'))
-
-                libdir = join(self.dist_dir, 'private', 'lib', 'python2.7')
-                site_packages_dir = join(libdir, 'site-packages')
-                with current_directory(libdir):
-                    removes = []
-                    for dirname, root, filenames in walk("."):
-                        for filename in filenames:
-                            for suffix in EXCLUDE_EXTS:
-                                if filename.endswith(suffix):
-                                    removes.append(filename)
-                    shprint(sh.rm, '-f', *removes)
-
-                    info('Deleting some other stuff not used on android')
-                    # To quote the original distribute.sh, 'well...'
-                    shprint(sh.rm, '-rf', 'lib2to3')
-                    shprint(sh.rm, '-rf', 'idlelib')
-                    for filename in glob.glob('config/libpython*.a'):
-                        shprint(sh.rm, '-f', filename)
-                    shprint(sh.rm, '-rf', 'config/python.o')
-
-            else:  # Python *is* loaded from crystax
-                ndk_dir = self.ctx.ndk_dir
-                py_recipe = self.ctx.python_recipe
-                python_dir = join(ndk_dir, 'sources', 'python',
-                                  py_recipe.version, 'libs', arch.arch)
-                shprint(sh.cp, '-r', join(python_dir,
-                                          'stdlib.zip'), crystax_python_dir)
-                shprint(sh.cp, '-r', join(python_dir,
-                                          'modules'), crystax_python_dir)
-                shprint(sh.cp, '-r', self.ctx.get_python_install_dir(),
-                        join(crystax_python_dir, 'site-packages'))
-
-                info('Renaming .so files to reflect cross-compile')
-                site_packages_dir = join(crystax_python_dir, "site-packages")
-                find_ret = shprint(
-                    sh.find, site_packages_dir, '-iname', '*.so')
-                filenames = find_ret.stdout.decode('utf-8').split('\n')[:-1]
-                for filename in filenames:
-                    parts = filename.split('.')
-                    if len(parts) <= 2:
-                        continue
-                    shprint(sh.mv, filename, filename.split('.')[0] + '.so')
-                site_packages_dir = join(abspath(curdir),
-                                         site_packages_dir)
+            python_bundle_dir = join('_python_bundle', '_python_bundle')
+            if 'python2' in self.ctx.recipe_build_order:
+                # Python 2 is a special case with its own packaging location
+                python_bundle_dir = 'private'
+            ensure_dir(python_bundle_dir)
+
+            site_packages_dir = self.ctx.python_recipe.create_python_bundle(
+                join(self.dist_dir, python_bundle_dir), arch)
+
             if 'sqlite3' not in self.ctx.recipe_build_order:
                 with open('blacklist.txt', 'a') as fileh:
                     fileh.write('\nsqlite3/*\nlib-dynload/_sqlite3.so\n')

pythonforandroid/bootstraps/sdl2/build/build.py

@@ -8,6 +8,7 @@
 import os
 import tarfile
 import time
+import json
 import subprocess
 import shutil
 from zipfile import ZipFile
@@ -125,7 +126,7 @@ def make_python_zip():
 
     if not exists('private'):
         print('No compiled python is present to zip, skipping.')
-        print('this should only be the case if you are using the CrystaX python')
+        print('this should only be the case if you are using the CrystaX python or python3')
         return
 
     global python_files
@@ -239,10 +240,9 @@ def make_package(args):
 
     # Package up the private data (public not supported).
     tar_dirs = [args.private]
-    if exists('private'):
-        tar_dirs.append('private')
-    if exists('crystax_python'):
-        tar_dirs.append('crystax_python')
+    for python_bundle_dir in ('private', 'crystax_python', '_python_bundle'):
+        if exists(python_bundle_dir):
+            tar_dirs.append(python_bundle_dir)
 
     if args.private:
         make_tar('src/main/assets/private.mp3', tar_dirs, args.ignore_path)
@@ -409,7 +409,17 @@ def make_package(args):
 
 def parse_args(args=None):
     global BLACKLIST_PATTERNS, WHITELIST_PATTERNS, PYTHON
-    default_android_api = 12
+
+    # Get the default minsdk, equal to the NDK API that this dist is built against
+    with open('dist_info.json', 'r') as fileh:
+        info = json.load(fileh)
+        if 'ndk_api' not in info:
+            print('WARNING: Failed to read ndk_api from dist info, defaulting to 12')
+            default_min_api = 12  # The old default before ndk_api was introduced
+        else:
+            default_min_api = info['ndk_api']
+            ndk_api = info['ndk_api']
+
     import argparse
     ap = argparse.ArgumentParser(description='''\
 Package a Python application for Android.
@@ -491,10 +501,13 @@ def parse_args(args=None):
     ap.add_argument('--sdk', dest='sdk_version', default=-1,
                     type=int, help=('Deprecated argument, does nothing'))
     ap.add_argument('--minsdk', dest='min_sdk_version',
-                    default=default_android_api, type=int,
-                    help=('Minimum Android SDK version to use. Default to '
-                          'the value of ANDROIDAPI, or {} if not set'
-                          .format(default_android_api)))
+                    default=default_min_api, type=int,
+                    help=('Minimum Android SDK version that the app supports. '
+                          'Defaults to {}.'.format(default_min_api)))
+    ap.add_argument('--allow-minsdk-ndkapi-mismatch', default=False,
+                    action='store_true',
+                    help=('Allow the --minsdk argument to be different from '
+                          'the discovered ndk_api in the dist'))
     ap.add_argument('--intent-filters', dest='intent_filters',
                     help=('Add intent-filters xml rules to the '
                           'AndroidManifest.xml file. The argument is a '
@@ -529,8 +542,18 @@ def parse_args(args=None):
     if args.name and args.name[0] == '"' and args.name[-1] == '"':
         args.name = args.name[1:-1]
 
-    # if args.sdk_version == -1:
-    #     args.sdk_version = args.min_sdk_version
+    if ndk_api != args.min_sdk_version:
+        print(('WARNING: --minsdk argument does not match the api that is '
+               'compiled against. Only proceed if you know what you are '
+               'doing, otherwise use --minsdk={} or recompile against api '
+               '{}').format(ndk_api, args.min_sdk_version))
+        if not args.allow_minsdk_ndkapi_mismatch:
+            print('You must pass --allow-minsdk-ndkapi-mismatch to build '
+                  'with --minsdk different to the target NDK api from the '
+                  'build step')
+            exit(1)
+        else:
+            print('Proceeding with --minsdk not matching build target api')
 
     if args.sdk_version != -1:
         print('WARNING: Received a --sdk argument, but this argument is '

pythonforandroid/bootstraps/sdl2/build/jni/Application.mk

@@ -5,3 +5,4 @@
 
 # APP_ABI := armeabi armeabi-v7a x86
 APP_ABI := $(ARCH)
+APP_PLATFORM := $(NDK_API)