Merge pull request #1457 from AndreMiras/feature/ci_improvements

Various CI improvements

Author: AndreMiras

.travis.yml

@@ -7,7 +7,7 @@ services:
 
 before_install:
   - sudo apt update -qq
-  - sudo apt install -qq python2.7 python3
+  - sudo apt install -qq --no-install-recommends python2.7 python3
   - sudo pip install tox>=2.0
   # https://github.com/travis-ci/travis-ci/issues/6069#issuecomment-266546552
   - git remote set-branches --add origin master

Dockerfile

@@ -1,16 +1,12 @@
 # Dockerfile with:
-# 	- Android build environment
-# 	- python-for-android dependencies
+#   - Android build environment
+#   - python-for-android dependencies
 # Build with:
 # docker build --tag=p4a .
 # Run with:
 # docker run p4a /bin/sh -c '. venv/bin/activate && p4a apk --help'
 # Or for interactive shell:
 # docker run -it --rm p4a
-#
-# TODO:
-#	- delete archives to keep small the container small
-#	- setup caching (for apt, pip, ndk, sdk and p4a recipes downloads)
 FROM ubuntu:18.04
 
 
@@ -27,35 +23,39 @@ ENV ANDROID_SDK_TOOLS_VERSION="3859397"
 
 ENV ANDROID_HOME="/opt/android"
 ENV ANDROID_NDK_HOME="${ANDROID_HOME}/android-ndk" \
-	CRYSTAX_NDK_HOME="${ANDROID_HOME}/crystax-ndk" \
-	ANDROID_SDK_HOME="${ANDROID_HOME}/android-sdk"
+    CRYSTAX_NDK_HOME="${ANDROID_HOME}/crystax-ndk" \
+    ANDROID_SDK_HOME="${ANDROID_HOME}/android-sdk"
 ENV ANDROID_NDK_HOME_V="${ANDROID_NDK_HOME}-r${ANDROID_NDK_VERSION}" \
-	CRYSTAX_NDK_HOME_V="${CRYSTAX_NDK_HOME}-${CRYSTAX_NDK_VERSION}"
+    CRYSTAX_NDK_HOME_V="${CRYSTAX_NDK_HOME}-${CRYSTAX_NDK_VERSION}"
 ENV ANDROID_NDK_ARCHIVE="android-ndk-r${ANDROID_NDK_VERSION}-linux-x86_64.zip" \
-	CRYSTAX_NDK_ARCHIVE="crystax-ndk-${CRYSTAX_NDK_VERSION}-linux-x86.tar.xz" \
-	ANDROID_SDK_TOOLS_ARCHIVE="sdk-tools-linux-${ANDROID_SDK_TOOLS_VERSION}.zip"
+    CRYSTAX_NDK_ARCHIVE="crystax-ndk-${CRYSTAX_NDK_VERSION}-linux-x86.tar.xz" \
+    ANDROID_SDK_TOOLS_ARCHIVE="sdk-tools-linux-${ANDROID_SDK_TOOLS_VERSION}.zip"
 ENV ANDROID_NDK_DL_URL="https://dl.google.com/android/repository/${ANDROID_NDK_ARCHIVE}" \
-	CRYSTAX_NDK_DL_URL="https://eu.crystax.net/download/${CRYSTAX_NDK_ARCHIVE}" \
-	ANDROID_SDK_TOOLS_DL_URL="https://dl.google.com/android/repository/${ANDROID_SDK_TOOLS_ARCHIVE}"
+    CRYSTAX_NDK_DL_URL="https://eu.crystax.net/download/${CRYSTAX_NDK_ARCHIVE}" \
+    ANDROID_SDK_TOOLS_DL_URL="https://dl.google.com/android/repository/${ANDROID_SDK_TOOLS_ARCHIVE}"
 
 # install system dependencies
 RUN apt update -qq && apt install -qq --yes --no-install-recommends \
-	python virtualenv python-pip wget curl lbzip2 patch bsdtar sudo && \
+    python virtualenv python-pip wget curl lbzip2 patch bsdtar sudo && \
     rm -rf /var/lib/apt/lists/*
 
 # build dependencies
 # https://buildozer.readthedocs.io/en/latest/installation.html#android-on-ubuntu-16-04-64bit
-RUN dpkg --add-architecture i386 &&  apt update -qq && apt install -qq --yes --no-install-recommends \
-	build-essential ccache git libncurses5:i386 libstdc++6:i386 libgtk2.0-0:i386 \
-	libpangox-1.0-0:i386 libpangoxft-1.0-0:i386 libidn11:i386 python2.7 python2.7-dev \
-	openjdk-8-jdk unzip zlib1g-dev zlib1g:i386 && \
+RUN dpkg --add-architecture i386 && apt update -qq && apt install -qq --yes --no-install-recommends \
+    build-essential ccache git libncurses5:i386 libstdc++6:i386 libgtk2.0-0:i386 \
+    libpangox-1.0-0:i386 libpangoxft-1.0-0:i386 libidn11:i386 python2.7 python2.7-dev \
+    openjdk-8-jdk unzip zlib1g-dev zlib1g:i386
+
+# specific recipes dependencies (e.g. libffi requires autoreconf binary)
+RUN apt install -qq --yes --no-install-recommends \
+    autoconf automake cmake gettext libltdl-dev libtool pkg-config && \
     rm -rf /var/lib/apt/lists/*
 
 # download and install Android NDK
 RUN curl --location --progress-bar "${ANDROID_NDK_DL_URL}" --output "${ANDROID_NDK_ARCHIVE}" && \
     mkdir --parents "${ANDROID_NDK_HOME_V}" && \
     unzip -q "${ANDROID_NDK_ARCHIVE}" -d "${ANDROID_HOME}" && \
-	ln -sfn "${ANDROID_NDK_HOME_V}" "${ANDROID_NDK_HOME}" && \
+    ln -sfn "${ANDROID_NDK_HOME_V}" "${ANDROID_NDK_HOME}" && \
     rm -rf "${ANDROID_NDK_ARCHIVE}"
 
 # download and install CrystaX NDK
@@ -70,7 +70,7 @@ RUN curl --location --progress-bar "${CRYSTAX_NDK_DL_URL}" --output "${CRYSTAX_N
     --exclude=crystax-ndk-${CRYSTAX_NDK_VERSION}/toolchains/llvm-* \
     --exclude=crystax-ndk-${CRYSTAX_NDK_VERSION}/toolchains/aarch64-* \
     --exclude=crystax-ndk-${CRYSTAX_NDK_VERSION}/toolchains/mips64el-* && \
-	ln -sfn "${CRYSTAX_NDK_HOME_V}" "${CRYSTAX_NDK_HOME}" && \
+    ln -sfn "${CRYSTAX_NDK_HOME_V}" "${CRYSTAX_NDK_HOME}" && \
     rm -rf "${CRYSTAX_NDK_ARCHIVE}"
 
 # download and install Android SDK
@@ -81,7 +81,7 @@ RUN curl --location --progress-bar "${ANDROID_SDK_TOOLS_DL_URL}" --output "${AND
 
 # update Android SDK, install Android API, Build Tools...
 RUN mkdir --parents "${ANDROID_SDK_HOME}/.android/" && \
-	echo '### User Sources for Android SDK Manager' > "${ANDROID_SDK_HOME}/.android/repositories.cfg"
+    echo '### User Sources for Android SDK Manager' > "${ANDROID_SDK_HOME}/.android/repositories.cfg"
 RUN yes | "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" --licenses
 RUN "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "platforms;android-19" && \
     "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "platforms;android-27" && \
@@ -93,7 +93,7 @@ RUN useradd --create-home --shell /bin/bash ${USER}
 # with sudo access and no password
 RUN usermod -append --groups sudo ${USER}
 RUN echo "%sudo ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
-RUN pip install --quiet --upgrade cython==0.21
+RUN pip install --quiet --upgrade Cython==0.28.6
 WORKDIR ${WORK_DIR}
 COPY . ${WORK_DIR}
 # user needs ownership/write access to these directories

ci/constants.py

@@ -29,25 +29,15 @@ class TargetPython(Enum):
     # https://github.com/kivy/python-for-android/issues/1354
     'kivent_core', 'kivent_cymunk', 'kivent_particles', 'kivent_polygen',
     'kiwisolver',
-    # system dependencies autoconf, libtool
-    'libexpat',
-    'libgeos',
     # https://github.com/kivy/python-for-android/issues/1399
     'libglob',
-    # system dependencies cmake and compile error
     'libmysqlclient',
     'libsecp256k1',
     'libtribler',
-    # system dependencies gettext, pkg-config
-    'libzbar',
     'ndghttpsclient',
     'm2crypto',
     'netifaces',
     'Pillow',
-    # requires autoconf system dependency on host
-    # https://api.travis-ci.org/v3/job/450538715/log.txt
-    'protobuf_cpp',
-    'cffi',
     # https://github.com/kivy/python-for-android/issues/1405
     'psycopg2',
     'pygame',
@@ -93,8 +83,6 @@ class TargetPython(Enum):
     'icu',
     # https://github.com/kivy/python-for-android/issues/1354
     'kivent_core', 'kivent_cymunk', 'kivent_particles', 'kivent_polygen',
-    # system dependencies autoconf, libtool
-    'libexpat',
     # https://github.com/kivy/python-for-android/issues/1405
     'libpq', 'psycopg2',
     'netifaces',

pythonforandroid/recipe.py

@@ -789,7 +789,7 @@ def install_python_package(self, arch, name=None, env=None, is_dir=True):
             hostpython = sh.Command(self.hostpython_location)
 
             if (self.ctx.python_recipe.from_crystax or
-                self.ctx.python_recipe.name == 'python3'):
+                    self.ctx.python_recipe.name == 'python3'):
                 hpenv = env.copy()
                 shprint(hostpython, 'setup.py', 'install', '-O2',
                         '--root={}'.format(self.ctx.get_python_install_dir()),

pythonforandroid/recipes/libffi/__init__.py

@@ -6,6 +6,12 @@
 
 
 class LibffiRecipe(Recipe):
+    """
+    Requires additional system dependencies on Ubuntu:
+        - `automake` for the `aclocal` binary
+        - `autoconf` for the `autoreconf` binary
+        - `libltdl-dev` which defines the `LT_SYS_SYMBOL_USCORE` macro
+    """
     name = 'libffi'
     version = 'v3.2.1'
     url = 'https://github.com/atgreen/libffi/archive/{version}.zip'

tox.ini

@@ -17,7 +17,6 @@ commands = flake8 pythonforandroid/ tests/ ci/
 ignore =
     E123, E124, E126,
     E226,
-    E129,
     E402, E501, E722,
     F812, F841, W503,
     W504