Conditional recipe build fixes #1382

.travis.yml

@@ -6,18 +6,27 @@ services:
   - docker
 
 before_install:
-  - sudo apt-get update -qq
-  - sudo apt-get install -qq python2.7 python3
+  - sudo apt update -qq
+  - sudo apt install -qq python2.7 python3
   - sudo pip install tox>=2.0
+  # https://github.com/travis-ci/travis-ci/issues/6069#issuecomment-266546552
+  - git remote set-branches --add origin master
+  - git fetch
 
 env:
-  - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python2.py apk --sdk-dir /opt/android/android-sdk --ndk-dir /opt/android/android-ndk'
-  # overrides requirements to skip `peewee` pure python module, see:
-  # https://github.com/kivy/python-for-android/issues/1263#issuecomment-390421054
-  - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python2_sqlite_openssl.py apk --sdk-dir /opt/android/android-sdk --ndk-dir /opt/android/android-ndk --requirements sdl2,pyjnius,kivy,python2,openssl,requests,sqlite3,setuptools'
-  - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python2.py apk --sdk-dir /opt/android/android-sdk --ndk-dir /opt/android/android-ndk --bootstrap sdl2 --requirements python2,numpy'
-  - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python3.py apk --sdk-dir /opt/android/android-sdk --ndk-dir /opt/android/crystax-ndk'
-  - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python3.py apk --sdk-dir /opt/android/android-sdk --ndk-dir /opt/android/crystax-ndk --requirements python3crystax,setuptools,android'
+  global:
+    - ANDROID_SDK_HOME=/opt/android/android-sdk
+    - ANDROID_NDK_HOME=/opt/android/android-ndk
+    - CRYSTAX_NDK_HOME=/opt/android/crystax-ndk
+  matrix:
+    - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python2.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME'
+    # overrides requirements to skip `peewee` pure python module, see:
+    # https://github.com/kivy/python-for-android/issues/1263#issuecomment-390421054
+    - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python2_sqlite_openssl.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME --requirements sdl2,pyjnius,kivy,python2,openssl,requests,sqlite3,setuptools'
+    - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python2.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME --bootstrap sdl2 --requirements python2,numpy'
+    - COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python3.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $CRYSTAX_NDK_HOME --requirements python3crystax,setuptools,android,sdl2,pyjnius,kivy'
+    # builds only the recipes that moved
+    - COMMAND='. venv/bin/activate && ./ci/rebuild_updated_recipes.py'
 
 before_script:
   # we want to fail fast on tox errors without having to `docker build` first

ci/rebuild_updated_recipes.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+Continuous Integration helper script.
+Automatically detects recipes modified in a changeset (compares with master)
+and recompiles them.
+
+To run locally, set the environment variables before running:
+```
+ANDROID_SDK_HOME=~/.buildozer/android/platform/android-sdk-20
+ANDROID_NDK_HOME=~/.buildozer/android/platform/android-ndk-r9c
+CRYSTAX_NDK_HOME=~/.buildozer/crystax-ndk
+./ci/rebuild_update_recipes.py
+```
+
+Current limitations:
+- handle the case with conflicting requirements
+  e.g. https://travis-ci.org/AndreMiras/python-for-android/builds/438840800
+  the list was huge and result was:
+  [ERROR]:   Didn't find any valid dependency graphs.
+  [ERROR]:   This means that some of your requirements pull in conflicting dependencies.
+- currently only builds on target python3crystax (even though python2 is supported)
+- comprehensive list of working/broken recipes is not yet known
+- only rebuilds on sdl2 bootstrap
+"""
+import sh
+import os
+from enum import Enum
+from pythonforandroid.toolchain import current_directory
+
+
+class TargetPython(Enum):
+    python2 = 0
+    python3crystax = 1
+
+
+# recipes that currently break the build
+# a recipe could be broken for a target Python and not for the other,
+# hence we're maintaining one list per Python target
+BROKEN_RECIPES_PYTHON2 = set([])
+BROKEN_RECIPES_PYTHON3_CRYSTAX = set([
+    # not yet python3crystax compatible
+    'apsw', 'atom', 'boost', 'brokenrecipe', 'cdecimal', 'cherrypy',
+    'coverage',
+    # https://github.com/kivy/python-for-android/issues/550
+    'audiostream',
+    # enum34 is not compatible with Python 3.6 standard library
+    # https://stackoverflow.com/a/45716067/185510
+    'enum34',
+    # https://github.com/kivy/python-for-android/issues/1398
+    'ifaddrs',
+    # https://github.com/kivy/python-for-android/issues/1399
+    'libglob',
+    # cannot find -lcrystax
+    'cffi',
+])
+BROKEN_RECIPES = {
+    TargetPython.python2: BROKEN_RECIPES_PYTHON2,
+    TargetPython.python3crystax: BROKEN_RECIPES_PYTHON3_CRYSTAX,
+}
+# recipes that are were already built
+CORE_RECIPES = set([
+    'pyjnius', 'kivy', 'openssl', 'requests', 'sqlite3', 'setuptools',
+    'numpy', 'android',
+])
+
+
+def modified_recipes(branch='origin/master'):
+    """
+    Returns a set of modified recipes between the current branch and the one
+    in param.
+    """
+    # using the contrib version on purpose rather than sh.git, since it comes
+    # with a bunch of fixes, e.g. disabled TTY, see:
+    # https://stackoverflow.com/a/20128598/185510
+    git_diff = sh.contrib.git.diff('--name-only', branch)
+    recipes = set()
+    for file_path in git_diff:
+        if 'pythonforandroid/recipes/' in file_path:
+            recipe = file_path.split('/')[2]
+            recipes.add(recipe)
+    return recipes
+
+
+def build(target_python, requirements):
+    """
+    Builds an APK given a target Python and a set of requirements.
+    """
+    if not requirements:
+        return
+    testapp = 'setup_testapp_python2.py'
+    android_sdk_home = os.environ['ANDROID_SDK_HOME']
+    android_ndk_home = os.environ['ANDROID_NDK_HOME']
+    crystax_ndk_home = os.environ['CRYSTAX_NDK_HOME']
+    if target_python == TargetPython.python3crystax:
+        android_ndk_home = crystax_ndk_home
+        testapp = 'setup_testapp_python3.py'
+    requirements.add(target_python.name)
+    requirements = ','.join(requirements)
+    print('requirements:', requirements)
+    with current_directory('testapps/'):
+        try:
+            for line in sh.python(
+                    testapp, 'apk', '--sdk-dir', android_sdk_home,
+                    '--ndk-dir', android_ndk_home, '--bootstrap', 'sdl2', '--requirements',
+                    requirements, _err_to_out=True, _iter=True):
+                print(line)
+        except sh.ErrorReturnCode as e:
+            raise
+
+
+def main():
+    target_python = TargetPython.python3crystax
+    recipes = modified_recipes()
+    print('recipes modified:', recipes)
+    broken_recipes = BROKEN_RECIPES[target_python]
+    recipes = recipes - (CORE_RECIPES | broken_recipes)
+    print('recipes to build:', recipes)
+    build(target_python, recipes)
+    print(recipes)
+
+
+if __name__ == '__main__':
+    main()

pythonforandroid/recipes/babel/__init__.py

@@ -3,7 +3,7 @@
 
 class BabelRecipe(PythonRecipe):
     name = 'babel'
-    version = '2.1.1'
+    version = '2.2.0'
     url = 'https://pypi.python.org/packages/source/B/Babel/Babel-{version}.tar.gz'
 
     depends = [('python2', 'python3crystax'), 'setuptools', 'pytz']

pythonforandroid/recipes/cffi/__init__.py

@@ -3,6 +3,9 @@
 
 
 class CffiRecipe(CompiledComponentsPythonRecipe):
+    """
+    Extra system dependencies: autoconf, automake and libtool.
+    """
     name = 'cffi'
     version = '1.4.2'
     url = 'https://pypi.python.org/packages/source/c/cffi/cffi-{version}.tar.gz'

setup.py

@@ -16,12 +16,9 @@
 data_files = []
 
 
-if os.name == 'nt':
-    install_reqs = ['appdirs', 'colorama>=0.3.3', 'jinja2',
-                        'six']
-else:
-    install_reqs = ['appdirs', 'colorama>=0.3.3', 'sh>=1.10', 'jinja2',
-                        'six']
+install_reqs = ['appdirs', 'colorama>=0.3.3', 'jinja2', 'six', 'enum34']
+if os.name != 'nt':
+    install_reqs.append('sh>=1.10')
 
 # By specifying every file manually, package_data will be able to
 # include them in binary distributions. Note that we have to add

tox.ini

@@ -11,7 +11,7 @@ commands = pytest {posargs:tests/}
 
 [testenv:pep8]
 deps = flake8
-commands = flake8 pythonforandroid/ tests/
+commands = flake8 pythonforandroid/ tests/ ci/
 
 [flake8]
 ignore =