Merge branch 'master' into omemo_recipe

Author: AndreMiras

.travis.yml

@@ -7,7 +7,7 @@ services:
 
 before_install:
   - sudo apt update -qq
-  - sudo apt install -qq python2.7 python3
+  - sudo apt install -qq --no-install-recommends python2.7 python3
   - sudo pip install tox>=2.0
   # https://github.com/travis-ci/travis-ci/issues/6069#issuecomment-266546552
   - git remote set-branches --add origin master
@@ -19,12 +19,12 @@ env:
     - ANDROID_NDK_HOME=/opt/android/android-ndk
     - CRYSTAX_NDK_HOME=/opt/android/crystax-ndk
   matrix:
-    - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python2.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME'
+    - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python3.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME --requirements sdl2,pyjnius,kivy,python3'
     # overrides requirements to skip `peewee` pure python module, see:
     # https://github.com/kivy/python-for-android/issues/1263#issuecomment-390421054
     - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python2_sqlite_openssl.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME --requirements sdl2,pyjnius,kivy,python2,openssl,requests,sqlite3,setuptools'
     - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python2.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME --bootstrap sdl2 --requirements python2,numpy'
-    - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python3.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $CRYSTAX_NDK_HOME --requirements python3crystax,setuptools,android,sdl2,pyjnius,kivy'
+    - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python3crystax.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $CRYSTAX_NDK_HOME --requirements python3crystax,setuptools,android,sdl2,pyjnius,kivy'
     # builds only the recipes that moved
     - COMMAND='. venv/bin/activate && ./ci/rebuild_updated_recipes.py'
 

Dockerfile

@@ -1,16 +1,12 @@
 # Dockerfile with:
-# 	- Android build environment
-# 	- python-for-android dependencies
+#   - Android build environment
+#   - python-for-android dependencies
 # Build with:
 # docker build --tag=p4a .
 # Run with:
 # docker run p4a /bin/sh -c '. venv/bin/activate && p4a apk --help'
 # Or for interactive shell:
 # docker run -it --rm p4a
-#
-# TODO:
-#	- delete archives to keep small the container small
-#	- setup caching (for apt, pip, ndk, sdk and p4a recipes downloads)
 FROM ubuntu:18.04
 
 
@@ -27,35 +23,39 @@ ENV ANDROID_SDK_TOOLS_VERSION="3859397"
 
 ENV ANDROID_HOME="/opt/android"
 ENV ANDROID_NDK_HOME="${ANDROID_HOME}/android-ndk" \
-	CRYSTAX_NDK_HOME="${ANDROID_HOME}/crystax-ndk" \
-	ANDROID_SDK_HOME="${ANDROID_HOME}/android-sdk"
+    CRYSTAX_NDK_HOME="${ANDROID_HOME}/crystax-ndk" \
+    ANDROID_SDK_HOME="${ANDROID_HOME}/android-sdk"
 ENV ANDROID_NDK_HOME_V="${ANDROID_NDK_HOME}-r${ANDROID_NDK_VERSION}" \
-	CRYSTAX_NDK_HOME_V="${CRYSTAX_NDK_HOME}-${CRYSTAX_NDK_VERSION}"
+    CRYSTAX_NDK_HOME_V="${CRYSTAX_NDK_HOME}-${CRYSTAX_NDK_VERSION}"
 ENV ANDROID_NDK_ARCHIVE="android-ndk-r${ANDROID_NDK_VERSION}-linux-x86_64.zip" \
-	CRYSTAX_NDK_ARCHIVE="crystax-ndk-${CRYSTAX_NDK_VERSION}-linux-x86.tar.xz" \
-	ANDROID_SDK_TOOLS_ARCHIVE="sdk-tools-linux-${ANDROID_SDK_TOOLS_VERSION}.zip"
+    CRYSTAX_NDK_ARCHIVE="crystax-ndk-${CRYSTAX_NDK_VERSION}-linux-x86.tar.xz" \
+    ANDROID_SDK_TOOLS_ARCHIVE="sdk-tools-linux-${ANDROID_SDK_TOOLS_VERSION}.zip"
 ENV ANDROID_NDK_DL_URL="https://dl.google.com/android/repository/${ANDROID_NDK_ARCHIVE}" \
-	CRYSTAX_NDK_DL_URL="https://eu.crystax.net/download/${CRYSTAX_NDK_ARCHIVE}" \
-	ANDROID_SDK_TOOLS_DL_URL="https://dl.google.com/android/repository/${ANDROID_SDK_TOOLS_ARCHIVE}"
+    CRYSTAX_NDK_DL_URL="https://eu.crystax.net/download/${CRYSTAX_NDK_ARCHIVE}" \
+    ANDROID_SDK_TOOLS_DL_URL="https://dl.google.com/android/repository/${ANDROID_SDK_TOOLS_ARCHIVE}"
 
 # install system dependencies
 RUN apt update -qq && apt install -qq --yes --no-install-recommends \
-	python virtualenv python-pip wget curl lbzip2 patch bsdtar sudo && \
+    python virtualenv python-pip wget curl lbzip2 patch bsdtar sudo && \
     rm -rf /var/lib/apt/lists/*
 
 # build dependencies
 # https://buildozer.readthedocs.io/en/latest/installation.html#android-on-ubuntu-16-04-64bit
-RUN dpkg --add-architecture i386 &&  apt update -qq && apt install -qq --yes --no-install-recommends \
-	build-essential ccache git libncurses5:i386 libstdc++6:i386 libgtk2.0-0:i386 \
-	libpangox-1.0-0:i386 libpangoxft-1.0-0:i386 libidn11:i386 python2.7 python2.7-dev \
-	openjdk-8-jdk unzip zlib1g-dev zlib1g:i386 && \
+RUN dpkg --add-architecture i386 && apt update -qq && apt install -qq --yes --no-install-recommends \
+    build-essential ccache git libncurses5:i386 libstdc++6:i386 libgtk2.0-0:i386 \
+    libpangox-1.0-0:i386 libpangoxft-1.0-0:i386 libidn11:i386 python2.7 python2.7-dev \
+    openjdk-8-jdk zip unzip zlib1g-dev zlib1g:i386
+
+# specific recipes dependencies (e.g. libffi requires autoreconf binary)
+RUN apt install -qq --yes --no-install-recommends \
+    autoconf automake cmake gettext libltdl-dev libtool pkg-config && \
     rm -rf /var/lib/apt/lists/*
 
 # download and install Android NDK
 RUN curl --location --progress-bar "${ANDROID_NDK_DL_URL}" --output "${ANDROID_NDK_ARCHIVE}" && \
     mkdir --parents "${ANDROID_NDK_HOME_V}" && \
     unzip -q "${ANDROID_NDK_ARCHIVE}" -d "${ANDROID_HOME}" && \
-	ln -sfn "${ANDROID_NDK_HOME_V}" "${ANDROID_NDK_HOME}" && \
+    ln -sfn "${ANDROID_NDK_HOME_V}" "${ANDROID_NDK_HOME}" && \
     rm -rf "${ANDROID_NDK_ARCHIVE}"
 
 # download and install CrystaX NDK
@@ -70,7 +70,7 @@ RUN curl --location --progress-bar "${CRYSTAX_NDK_DL_URL}" --output "${CRYSTAX_N
     --exclude=crystax-ndk-${CRYSTAX_NDK_VERSION}/toolchains/llvm-* \
     --exclude=crystax-ndk-${CRYSTAX_NDK_VERSION}/toolchains/aarch64-* \
     --exclude=crystax-ndk-${CRYSTAX_NDK_VERSION}/toolchains/mips64el-* && \
-	ln -sfn "${CRYSTAX_NDK_HOME_V}" "${CRYSTAX_NDK_HOME}" && \
+    ln -sfn "${CRYSTAX_NDK_HOME_V}" "${CRYSTAX_NDK_HOME}" && \
     rm -rf "${CRYSTAX_NDK_ARCHIVE}"
 
 # download and install Android SDK
@@ -81,9 +81,10 @@ RUN curl --location --progress-bar "${ANDROID_SDK_TOOLS_DL_URL}" --output "${AND
 
 # update Android SDK, install Android API, Build Tools...
 RUN mkdir --parents "${ANDROID_SDK_HOME}/.android/" && \
-	echo '### User Sources for Android SDK Manager' > "${ANDROID_SDK_HOME}/.android/repositories.cfg"
+    echo '### User Sources for Android SDK Manager' > "${ANDROID_SDK_HOME}/.android/repositories.cfg"
 RUN yes | "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" --licenses
 RUN "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "platforms;android-19" && \
+    "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "platforms;android-27" && \
     "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "build-tools;26.0.2" && \
     chmod +x "${ANDROID_SDK_HOME}/tools/bin/avdmanager"
 
@@ -92,7 +93,7 @@ RUN useradd --create-home --shell /bin/bash ${USER}
 # with sudo access and no password
 RUN usermod -append --groups sudo ${USER}
 RUN echo "%sudo ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
-RUN pip install --quiet --upgrade cython==0.21
+RUN pip install --quiet --upgrade cython==0.28.6
 WORKDIR ${WORK_DIR}
 COPY . ${WORK_DIR}
 # user needs ownership/write access to these directories

ci/constants.py

@@ -29,29 +29,23 @@ class TargetPython(Enum):
     # https://github.com/kivy/python-for-android/issues/1354
     'kivent_core', 'kivent_cymunk', 'kivent_particles', 'kivent_polygen',
     'kiwisolver',
-    # system dependencies autoconf, libtool
-    'libexpat',
-    'libgeos',
     # https://github.com/kivy/python-for-android/issues/1399
     'libglob',
-    # system dependencies cmake and compile error
     'libmysqlclient',
     'libsecp256k1',
     'libtribler',
-    # system dependencies gettext, pkg-config
-    'libzbar',
     'ndghttpsclient',
     'm2crypto',
     'netifaces',
     'Pillow',
-    # requires autoconf system dependency on host
-    # https://api.travis-ci.org/v3/job/450538715/log.txt
+    # depends on cffi that still seems to have compilation issues
     'protobuf_cpp',
     'xeddsa',
     'x3dh',
     'pynacl',
     'doubleratchet',
     'omemo',
+    'cryptography',
     # https://github.com/kivy/python-for-android/issues/1405
     'psycopg2',
     'pygame',
@@ -69,16 +63,7 @@ class TargetPython(Enum):
     'zope',
 ])
 BROKEN_RECIPES_PYTHON3_CRYSTAX = set([
-    # not yet python3crystax compatible
-    'apsw', 'atom', 'boost', 'brokenrecipe', 'cdecimal', 'cherrypy',
-    'coverage', 'dateutil', 'enaml', 'ethash', 'kiwisolver', 'libgeos',
-    'libnacl', 'libsodium', 'libtorrent', 'libtribler', 'libzbar', 'libzmq',
-    'm2crypto', 'mysqldb', 'ndghttpsclient', 'pil', 'pycrypto', 'pyethereum',
-    'pygame', 'pyleveldb', 'pyproj', 'pyzmq', 'regex', 'shapely',
-    'simple-crypt', 'twsisted', 'vispy', 'websocket-client', 'zbar',
-    'zeroconf', 'zope',
-    # https://github.com/kivy/python-for-android/issues/550
-    'audiostream',
+    'brokenrecipe',
     # enum34 is not compatible with Python 3.6 standard library
     # https://stackoverflow.com/a/45716067/185510
     'enum34',
@@ -97,8 +82,6 @@ class TargetPython(Enum):
     'icu',
     # https://github.com/kivy/python-for-android/issues/1354
     'kivent_core', 'kivent_cymunk', 'kivent_particles', 'kivent_polygen',
-    # system dependencies autoconf, libtool
-    'libexpat',
     # https://github.com/kivy/python-for-android/issues/1405
     'libpq', 'psycopg2',
     'netifaces',

doc/source/buildoptions.rst

@@ -17,16 +17,23 @@ This option builds Python 2.7.2 for your selected Android
 architecture. There are no special requirements, all the building is
 done locally.
 
-The python2 build is also the way python-for-android originally
-worked, even in the old toolchain.
-
 
 python3
 ~~~~~~~
 
-.. warning::
-   Python3 support is experimental, and some of these details
-   may change as it is improved and fully stabilised.
+Python3 is supported in two ways. The default method uses CPython 3.7+
+and works with any recent version of the Android NDK.
+
+Select Python 3 by adding it to your requirements,
+e.g. ``--requirements=python3``.
+
+
+CrystaX python3
+###############
+
+.. warning:: python-for-android originally supported Python 3 using the CrystaX
+             NDK. This support is now being phased out as CrystaX is no longer
+             actively developed.
 
 .. note:: You must manually download the `CrystaX NDK
    <https://www.crystax.net/android/ndk>`__ and tell
@@ -42,11 +49,6 @@ Google's official NDK which includes many improvements. You
 python3. You can get it `here
 <https://www.crystax.net/en/download>`__.
 
-The python3crystax build is handled quite differently to python2 so
-there may be bugs or surprising behaviours. If you come across any,
-feel free to `open an issue
-<https://github.com/kivy/python-for-android>`__.
-
 .. _bootstrap_build_options:
 
 Bootstrap options

doc/source/quickstart.rst

@@ -114,15 +114,17 @@ Then, you can edit your ``~/.bashrc`` or other favorite shell to include new env
     # Adjust the paths!
     export ANDROIDSDK="$HOME/Documents/android-sdk-21"
     export ANDROIDNDK="$HOME/Documents/android-ndk-r10e"
-    export ANDROIDAPI="19"  # Target API version of your application
+    export ANDROIDAPI="26"  # Target API version of your application
+    export NDKAPI="19"  # Minimum supported API version of your application
     export ANDROIDNDKVER="r10e"  # Version of the NDK you installed
 
 You have the possibility to configure on any command the PATH to the SDK, NDK and Android API using:
 
-- :code:`--sdk_dir PATH` as an equivalent of `$ANDROIDSDK`
-- :code:`--ndk_dir PATH` as an equivalent of `$ANDROIDNDK`
-- :code:`--android_api VERSION` as an equivalent of `$ANDROIDAPI`
-- :code:`--ndk_version VERSION` as an equivalent of `$ANDROIDNDKVER`
+- :code:`--sdk-dir PATH` as an equivalent of `$ANDROIDSDK`
+- :code:`--ndk-dir PATH` as an equivalent of `$ANDROIDNDK`
+- :code:`--android-api VERSION` as an equivalent of `$ANDROIDAPI`
+- :code:`--ndk-api VERSION` as an equivalent of `$NDKAPI`
+- :code:`--ndk-version VERSION` as an equivalent of `$ANDROIDNDKVER`
 
 
 Usage

pythonforandroid/archs.py

@@ -35,7 +35,7 @@ def get_env(self, with_flags_in_cc=True):
 
         env['CFLAGS'] = ' '.join([
             '-DANDROID', '-mandroid', '-fomit-frame-pointer'
-            ' -D__ANDROID_API__={}'.format(self.ctx._android_api),
+            ' -D__ANDROID_API__={}'.format(self.ctx.ndk_api),
             ])
         env['LDFLAGS'] = ' '
 
@@ -133,6 +133,7 @@ def get_env(self, with_flags_in_cc=True):
         env['PATH'] = environ['PATH']
 
         env['ARCH'] = self.arch
+        env['NDK_API'] = 'android-{}'.format(str(self.ctx.ndk_api))
 
         if self.ctx.python_recipe and self.ctx.python_recipe.from_crystax:
             env['CRYSTAX_PYTHON_VERSION'] = self.ctx.python_recipe.version

pythonforandroid/bootstrap.py

@@ -2,7 +2,6 @@
 from os import listdir
 import sh
 import glob
-import json
 import importlib
 
 from pythonforandroid.logger import (warning, shprint, info, logger,
@@ -102,21 +101,10 @@ def prepare_build_dir(self):
                 fileh.write('target=android-{}'.format(self.ctx.android_api))
 
     def prepare_dist_dir(self, name):
-        # self.dist_dir = self.get_dist_dir(name)
         ensure_dir(self.dist_dir)
 
     def run_distribute(self):
-        # print('Default bootstrap being used doesn\'t know how '
-        #       'to distribute...failing.')
-        # exit(1)
-        with current_directory(self.dist_dir):
-            info('Saving distribution info')
-            with open('dist_info.json', 'w') as fileh:
-                json.dump({'dist_name': self.ctx.dist_name,
-                           'bootstrap': self.ctx.bootstrap.name,
-                           'archs': [arch.arch for arch in self.ctx.archs],
-                           'recipes': self.ctx.recipe_build_order + self.ctx.python_modules},
-                          fileh)
+        self.distribution.save_info(self.dist_dir)
 
     @classmethod
     def list_bootstraps(cls):
@@ -254,9 +242,15 @@ def strip_libraries(self, arch):
             warning('Can\'t find strip in PATH...')
             return
         strip = sh.Command(strip)
-        filens = shprint(sh.find, join(self.dist_dir, 'private'),
-                         join(self.dist_dir, 'libs'),
-                         '-iname', '*.so', _env=env).stdout.decode('utf-8')
+
+        if self.ctx.python_recipe.name == 'python2':
+            filens = shprint(sh.find, join(self.dist_dir, 'private'),
+                             join(self.dist_dir, 'libs'),
+                             '-iname', '*.so', _env=env).stdout.decode('utf-8')
+        else:
+            filens = shprint(sh.find, join(self.dist_dir, '_python_bundle', '_python_bundle', 'modules'),
+                             join(self.dist_dir, 'libs'),
+                             '-iname', '*.so', _env=env).stdout.decode('utf-8')
         logger.info('Stripping libraries in private dir')
         for filen in filens.split('\n'):
             try: