Refactor python recipes and update python2's to version 2.7.15 (fixed history)

.travis.yml

@@ -18,11 +18,14 @@ env:
     - ANDROID_SDK_HOME=/opt/android/android-sdk
     - ANDROID_NDK_HOME=/opt/android/android-ndk
   matrix:
-    - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python3.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME --requirements sdl2,pyjnius,kivy,python3'
+    # test encryption recipes
+    - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python_encryption.py apk --dist-name bdisttest_python3_encryption --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME --requirements python3,libffi,openssl,sdl2,pyjnius,kivy,cryptography,pycrypto,scrypt,m2crypto,pysha3,pycryptodome,libtorrent'
     # overrides requirements to skip `peewee` pure python module, see:
     # https://github.com/kivy/python-for-android/issues/1263#issuecomment-390421054
+    - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python3_sqlite_openssl.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME --requirements libffi,sdl2,pyjnius,kivy,python3,openssl,requests,sqlite3,setuptools'
     - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python2_sqlite_openssl.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME --requirements sdl2,pyjnius,kivy,python2,openssl,requests,sqlite3,setuptools'
-    - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python2.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME --bootstrap sdl2 --requirements python2,numpy'
+    # test webview bootstrap (plus numpy)
+    - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_flask.py apk --dist-name bdisttest_python3_flask_webview --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME --requirements python3,flask,pyjnius,numpy'
     # builds only the recipes that moved
     - COMMAND='. venv/bin/activate && ./ci/rebuild_updated_recipes.py'
 
@@ -31,5 +34,14 @@ before_script:
   - tox
 
 script:
-  - docker build --tag=p4a --file Dockerfile.py2 .
-  - docker run p4a /bin/sh -c "$COMMAND"
+  # Capture any error we got while doing our builds
+  - trap "echo \"Error...last docker's log build lines:\" & tail -500 build.log & echo \"Error last tests log lines:\" &tail -1000 tests.log" ERR
+  # The Docker image build
+  - while sleep 1m; do echo "=====[ $SECONDS seconds, docker image still building... ]====="; done & PING_LOOP1_PID=$!
+  - docker build --tag=p4a --file Dockerfile.py3 . > build.log 2>&1
+  - kill ${PING_LOOP1_PID}
+  # The Tests build
+  - while sleep 1m; do echo "=====[ $SECONDS seconds, tests still building... ]====="; done & PING_LOOP2_PID=$!
+  - docker run p4a /bin/sh -c "$COMMAND" > tests.log 2>&1
+  - kill ${PING_LOOP2_PID}
+  - echo "All operations performed...last build log lines are:" & tail -500 tests.log

.travis.yml.unused

@@ -0,0 +1,11 @@
+# Here we put some tests that could be enabled, depending on the p4a project needs
+
+# Test encryption recipes
+- COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python_encryption.py apk --dist-name bdisttest_python2_encryption --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME --requirements python2,libffi,openssl,sdl2,pyjnius,kivy,cryptography,pycrypto,scrypt,m2crypto,pysha3,pycryptodome,libtorrent'
+
+# test webview bootstrap (plus numpy)
+- COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_flask.py apk --dist-name bdisttest_python2_flask_webview --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME --requirements python2,flask,pyjnius,numpy'
+
+# test service only bootstrap (plus numpy)
+- COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_service.py apk --dist-name bdisttest_python3_service --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME --requirements python3,genericndkbuild,numpy'
+- COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_service.py apk --dist-name bdisttest_python2_service --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME --requirements python2,genericndkbuild,numpy'

Dockerfile.py2

@@ -26,7 +26,7 @@ RUN apt -y update -qq \
 
 
 ENV ANDROID_NDK_HOME="${ANDROID_HOME}/android-ndk"
-ENV ANDROID_NDK_VERSION="16b"
+ENV ANDROID_NDK_VERSION="17c"
 ENV ANDROID_NDK_HOME_V="${ANDROID_NDK_HOME}-r${ANDROID_NDK_VERSION}"
 
 # get the latest version from https://developer.android.com/ndk/downloads/index.html
@@ -85,7 +85,7 @@ ENV WORK_DIR="${HOME_DIR}" \
 # install system dependencies
 RUN apt -y update -qq \
     && apt -y install -qq --no-install-recommends \
-        python virtualenv python-pip wget lbzip2 patch sudo \
+        python virtualenv python-pip wget lbzip2 patch sudo swig \
     && apt -y autoremove \
     && apt -y clean
 
@@ -104,7 +104,7 @@ RUN dpkg --add-architecture i386 \
 # specific recipes dependencies (e.g. libffi requires autoreconf binary)
 RUN apt -y update -qq \
     && apt -y install -qq --no-install-recommends \
-        autoconf automake cmake gettext libltdl-dev libtool pkg-config \
+        libffi-dev autoconf automake cmake gettext libltdl-dev libtool pkg-config \
     && apt -y autoremove \
     && apt -y clean
 

Dockerfile.py3

@@ -15,7 +15,7 @@
 #     Use 'docker run' without '--rm' flag for keeping the container and use
 #     'docker commit <container hash> <new image>' to extend the original image
 
-FROM ubuntu:18.04
+FROM python:3.7-stretch
 
 ENV ANDROID_HOME="/opt/android"
 
@@ -26,7 +26,7 @@ RUN apt -y update -qq \
 
 
 ENV ANDROID_NDK_HOME="${ANDROID_HOME}/android-ndk"
-ENV ANDROID_NDK_VERSION="16b"
+ENV ANDROID_NDK_VERSION="17c"
 ENV ANDROID_NDK_HOME_V="${ANDROID_NDK_HOME}-r${ANDROID_NDK_VERSION}"
 
 # get the latest version from https://developer.android.com/ndk/downloads/index.html
@@ -85,7 +85,7 @@ ENV WORK_DIR="${HOME_DIR}" \
 # install system dependencies
 RUN apt -y update -qq \
     && apt -y install -qq --no-install-recommends \
-        python3 virtualenv python3-pip wget lbzip2 patch sudo \
+        python3 virtualenv python3-pip wget lbzip2 patch sudo swig \
     && apt -y autoremove \
     && apt -y clean
 
@@ -104,7 +104,7 @@ RUN dpkg --add-architecture i386 \
 # specific recipes dependencies (e.g. libffi requires autoreconf binary)
 RUN apt -y update -qq \
     && apt -y install -qq --no-install-recommends \
-        autoconf automake cmake gettext libltdl-dev libtool pkg-config \
+        libffi-dev autoconf automake cmake gettext libltdl-dev libtool pkg-config \
     && apt -y autoremove \
     && apt -y clean
 

README.rst

@@ -35,6 +35,20 @@ issues and PRs relating to this branch are still accepted. However, the
 new toolchain contains all the same functionality via the built in
 pygame bootstrap.
 
+In the last quarter of 2018 the python recipes has been changed, the new recipe
+for python3 (3.7.1) has a new build system which has been applied to the ancient
+python recipe, allowing us to bump the python2 version number to 2.7.15. This
+change, unifies the build process for both python recipes, and probably solve
+some issues detected over the years, but unfortunately, this change breaks the
+pygame bootstrap (in a near future we will fix it or remove it). This also
+means that the ndk version to use in python-for-android should be at least the
+version r17c.
+
+This has been done this way to make easier the transition between python3 and
+python2. We will try to support python2, at least until the python team release
+the last python2 version (2020), but if you are using python2 in your projects
+you should consider to migrate it into python3.
+
 Documentation
 =============
 

doc/source/quickstart.rst

@@ -93,11 +93,34 @@ named ``tools``, and you will need to run extra commands to install
 the SDK packages needed. 
 
 For Android NDK, note that modern releases will only work on a 64-bit
-operating system. If you are using a 32-bit distribution (or hardware),
-the latest useable NDK version is r10e, which can be downloaded here:
+operating system. The minimal, and recommended, NDK version to use is r17c:
+
+ - `Linux (64-bits) <https://dl.google.com/android/repository/android-ndk-r17c-linux-x86_64.zip>`_
+ - `Macos X (64-bits) <https://dl.google.com/android/repository/android-ndk-r17c-darwin-x86_64.zip>`_
+ - Windows users should create a virtual machine with an GNU Linux os
+   installed, and then you can follow the described instructions from within
+   your virtual machine.
+
+.. note::
+    Versions below r17 may cause troubles while compiling some recipes. Our
+    tests had been done against ndk versions r17b and r17c. Later versions of
+    the android ndk still are not proved to work with python-for-android, and
+    may work or not...so...better to stick to the recommended version (r17c).
+
+If you are using a 32-bit distribution (or hardware),
+the latest usable NDK version is r10e, which can be downloaded here:
 
 - `Legacy 32-bit Linux NDK r10e <http://dl.google.com/android/ndk/android-ndk-r10e-linux-x86.bin>`_
 
+.. warning::
+    **32-bit distributions**
+
+    Since the python2 recipe updated to version 2.7.15, the build system has
+    been changed and you should use an old release of python-for-android, which
+    contains the legacy python recipe (v2.7.2). The last python-for-android
+    release with the legacy version of python is version
+    `0.6.0 <https://github.com/kivy/python-for-android/archive/0.6.0.zip>`_.
+
 First, install a platform to target (you can also replace ``27`` with
 a different platform number, this will be used again later)::
 
@@ -113,8 +136,8 @@ Then, you can edit your ``~/.bashrc`` or other favorite shell to include new env
 
     # Adjust the paths!
     export ANDROIDSDK="$HOME/Documents/android-sdk-27"
-    export ANDROIDNDK="$HOME/Documents/android-ndk-r10e"
-    export ANDROIDAPI="27"  # Target API version of your application
+    export ANDROIDNDK="$HOME/Documents/android-ndk-r17c"
+    export ANDROIDAPI="26"  # Target API version of your application
     export NDKAPI="19"  # Minimum supported API version of your application
     export ANDROIDNDKVER="r10e"  # Version of the NDK you installed
 

pythonforandroid/archs.py

@@ -30,13 +30,24 @@ def include_dirs(self):
                 d.format(arch=self))
             for d in self.ctx.include_dirs]
 
-    def get_env(self, with_flags_in_cc=True):
+    def get_env(self, with_flags_in_cc=True, clang=False):
         env = {}
 
-        env['CFLAGS'] = ' '.join([
-            '-DANDROID', '-mandroid', '-fomit-frame-pointer'
-            ' -D__ANDROID_API__={}'.format(self.ctx.ndk_api),
-            ])
+        cflags = [
+            '-DANDROID',
+            '-fomit-frame-pointer',
+            '-D__ANDROID_API__={}'.format(self.ctx.ndk_api)]
+        if not clang:
+            cflags += ['-mandroid']
+        else:
+            cflags += ['-target armv7-none-linux-androideabi']
+            android_host = 'arm-linux-androideabi'
+            platform_dir = join(self.ctx.ndk_dir, 'platforms', 'android-{}'.format(self.ctx.ndk_api), 'arch-arm')
+            toolchain = '{android_host}-4.9'.format(android_host=android_host)
+            toolchain = join(self.ctx.ndk_dir, 'toolchains', toolchain, 'prebuilt', 'linux-x86_64')
+            cflags += ['-gcc-toolchain {}'.format(toolchain)]
+
+        env['CFLAGS'] = ' '.join(cflags)
         env['LDFLAGS'] = ' '
 
         sysroot = join(self.ctx._ndk_dir, 'sysroot')
@@ -82,8 +93,19 @@ def get_env(self, with_flags_in_cc=True):
             env['NDK_CCACHE'] = self.ctx.ccache
             env.update({k: v for k, v in environ.items() if k.startswith('CCACHE_')})
 
-        cc = find_executable('{command_prefix}-gcc'.format(
-            command_prefix=command_prefix), path=environ['PATH'])
+        if clang:
+            clang_path = join(
+                self.ctx.ndk_dir, 'toolchains', 'llvm', 'prebuilt',
+                'linux-x86_64', 'bin')
+            environ['PATH'] = '{clang_path}:{path}'.format(
+                clang_path=clang_path, path=environ['PATH'])
+            exe = join(clang_path, 'clang')
+            execxx = join(clang_path, 'clang++')
+        else:
+            exe = '{command_prefix}-gcc'.format(command_prefix=command_prefix)
+            execxx = '{command_prefix}-g++'.format(command_prefix=command_prefix)
+
+        cc = find_executable(exe, path=environ['PATH'])
         if cc is None:
             print('Searching path are: {!r}'.format(environ['PATH']))
             raise BuildInterruptingException(
@@ -93,20 +115,20 @@ def get_env(self, with_flags_in_cc=True):
                 'installed. Exiting.')
 
         if with_flags_in_cc:
-            env['CC'] = '{ccache}{command_prefix}-gcc {cflags}'.format(
-                command_prefix=command_prefix,
+            env['CC'] = '{ccache}{exe} {cflags}'.format(
+                exe=exe,
                 ccache=ccache,
                 cflags=env['CFLAGS'])
-            env['CXX'] = '{ccache}{command_prefix}-g++ {cxxflags}'.format(
-                command_prefix=command_prefix,
+            env['CXX'] = '{ccache}{execxx} {cxxflags}'.format(
+                execxx=execxx,
                 ccache=ccache,
                 cxxflags=env['CXXFLAGS'])
         else:
-            env['CC'] = '{ccache}{command_prefix}-gcc'.format(
-                command_prefix=command_prefix,
+            env['CC'] = '{ccache}{exe}'.format(
+                exe=exe,
                 ccache=ccache)
-            env['CXX'] = '{ccache}{command_prefix}-g++'.format(
-                command_prefix=command_prefix,
+            env['CXX'] = '{ccache}{execxx}'.format(
+                execxx=execxx,
                 ccache=ccache)
 
         env['AR'] = '{}-ar'.format(command_prefix)
@@ -123,12 +145,13 @@ def get_env(self, with_flags_in_cc=True):
         env['READELF'] = '{}-readelf'.format(command_prefix)
         env['NM'] = '{}-nm'.format(command_prefix)
 
-        hostpython_recipe = Recipe.get_recipe('hostpython2', self.ctx)
-
-        # This hardcodes python version 2.7, needs fixing
+        hostpython_recipe = Recipe.get_recipe(
+            'host' + self.ctx.python_recipe.name, self.ctx)
         env['BUILDLIB_PATH'] = join(
             hostpython_recipe.get_build_dir(self.arch),
-            'build', 'lib.linux-{}-2.7'.format(uname()[-1]))
+            'build', 'lib.linux-{}-{}'.format(
+                uname()[-1], self.ctx.python_recipe.major_minor_version_string)
+        )
 
         env['PATH'] = environ['PATH']
 
@@ -151,8 +174,8 @@ class ArchARM(Arch):
 class ArchARMv7_a(ArchARM):
     arch = 'armeabi-v7a'
 
-    def get_env(self, with_flags_in_cc=True):
-        env = super(ArchARMv7_a, self).get_env(with_flags_in_cc)
+    def get_env(self, with_flags_in_cc=True, clang=False):
+        env = super(ArchARMv7_a, self).get_env(with_flags_in_cc, clang=clang)
         env['CFLAGS'] = (env['CFLAGS'] +
                          (' -march=armv7-a -mfloat-abi=softfp '
                           '-mfpu=vfp -mthumb'))
@@ -166,8 +189,8 @@ class Archx86(Arch):
     command_prefix = 'i686-linux-android'
     platform_dir = 'arch-x86'
 
-    def get_env(self, with_flags_in_cc=True):
-        env = super(Archx86, self).get_env(with_flags_in_cc)
+    def get_env(self, with_flags_in_cc=True, clang=False):
+        env = super(Archx86, self).get_env(with_flags_in_cc, clang=clang)
         env['CFLAGS'] = (env['CFLAGS'] +
                          ' -march=i686 -mtune=intel -mssse3 -mfpmath=sse -m32')
         env['CXXFLAGS'] = env['CFLAGS']
@@ -180,8 +203,8 @@ class Archx86_64(Arch):
     command_prefix = 'x86_64-linux-android'
     platform_dir = 'arch-x86_64'
 
-    def get_env(self, with_flags_in_cc=True):
-        env = super(Archx86_64, self).get_env(with_flags_in_cc)
+    def get_env(self, with_flags_in_cc=True, clang=False):
+        env = super(Archx86_64, self).get_env(with_flags_in_cc, clang=clang)
         env['CFLAGS'] = (env['CFLAGS'] +
                          ' -march=x86-64 -msse4.2 -mpopcnt -m64 -mtune=intel')
         env['CXXFLAGS'] = env['CFLAGS']
@@ -194,8 +217,8 @@ class ArchAarch_64(Arch):
     command_prefix = 'aarch64-linux-android'
     platform_dir = 'arch-arm64'
 
-    def get_env(self, with_flags_in_cc=True):
-        env = super(ArchAarch_64, self).get_env(with_flags_in_cc)
+    def get_env(self, with_flags_in_cc=True, clang=False):
+        env = super(ArchAarch_64, self).get_env(with_flags_in_cc, clang=clang)
         incpath = ' -I' + join(dirname(__file__), 'includes', 'arm64-v8a')
         env['EXTRA_CFLAGS'] = incpath
         env['CFLAGS'] += incpath