Travis CI revamp part 1, refs #2008

AndreMiras · AndreMiras · commit a9750156c183 · 2019-10-27T13:08:00.000+01:00
Revamping Travis and Docker setup introducing a `Makefile`. The idea is to move the CI complexity from .travis.yml to `Makefile`. That makes a single entry point via `make` command and reproducible builds via Docker. It makes it easy to run some commands outside docker, such as: ```sh make testapps/python3/armeabi-v7a ``` Or the same command inside docker: ```sh make docker/run/make/testapps/python3/armeabi-v7a ``` This pull request also starts introducing some docker layer cache optimization as needed by #2009 to speed up docker pull/push and rebuilds from cache. It also introduces other Docker images good practices like ordering dependencies alphabetically or always enforcing `apt update` prior install, refs: https://docs.docker.com/develop/develop-images/dockerfile_best-practices/ Subsequent pull requests would simplify the process furthermore and leverage the cache to speed up builds.
diff --git a/.env b/.env
@@ -0,0 +1,9 @@
+# used by coveralls.io, refs:
+# https://coveralls-python.readthedocs.io/en/latest/usage/tox.html#travisci
+CI
+TRAVIS
+TRAVIS_BRANCH
+TRAVIS_JOB_ID
+TRAVIS_PULL_REQUEST
+# used for running UI tests
+DISPLAY
diff --git a/.travis.yml b/.travis.yml
@@ -1,12 +1,10 @@
 sudo: required
 
-dist: xenial  # needed for more recent python 3 and python3-venv
-
 language: generic
 
 stages:
-  - lint
-  - test
+  - unit tests
+  - build apps
 
 services:
   - docker
@@ -18,8 +16,8 @@ before_install:
 
 jobs:
   include:
-    - &linting
-      stage: lint
+    - &unittests
+      stage: unit tests
       language: python
       python: 3.7
       before_script:
@@ -36,46 +34,26 @@ jobs:
         - pip3.7 install pyOpenSSL
         - pip3.7 install coveralls
       script:
-        # we want to fail fast on tox errors without having to `docker build` first
+        # ignores test_pythonpackage.py since it runs for too long
         - tox -- tests/ --ignore tests/test_pythonpackage.py
-        # (we ignore test_pythonpackage.py since these run way too long!!
-        #  test_pythonpackage_basic.py will still be run.)
       name: "Tox Pep8"
       env: TOXENV=pep8
-    - <<: *linting
+    - <<: *unittests
       name: "Tox Python 2"
       env: TOXENV=py27
-    - <<: *linting
+    - <<: *unittests
       name: "Tox Python 3 & Coverage"
       env: TOXENV=py3
       after_success:
         - coveralls
-
     - &testing
-      stage: test
+      stage: build apps
       before_script:
-        # build docker image
-        - docker build --tag=p4a --file Dockerfile.py3 .
-        # Run a background process to make sure that travis will not kill our tests in
-        # case that the travis log doesn't produce any output for more than 10 minutes
-        - while sleep 540; do echo "==== Still running (travis, don't kill me) ===="; done &
-      script:
-        - >
-          docker run
-          -e CI
-          -e TRAVIS_JOB_ID
-          -e TRAVIS_BRANCH
-          -e ANDROID_SDK_HOME="/home/user/.android/android-sdk"
-          -e ANDROID_NDK_HOME="/home/user/.android/android-ndk"
-          p4a /bin/sh -c "$COMMAND"
-      after_script:
-        # kill the background process started before run docker
-        - kill %1
+        # ideally we would have a stage for that so it's not ran multiple time
+        # but it seems Travis doesn't share the build cache
+        - make docker/build
       name: Python 3 arm64-v8a
-      # overrides requirements to skip `peewee` pure python module, see:
-      # https://github.com/kivy/python-for-android/issues/1263#issuecomment-390421054
-      env:
-        COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python3_sqlite_openssl.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME --requirements libffi,sdl2,pyjnius,kivy,python3,openssl,requests,sqlite3,setuptools --arch=arm64-v8a'
+      script: make docker/run/make/testapps/python3/arm64-v8a
     - <<: *testing
       name: Python 3 armeabi-v7a
       os: osx
@@ -84,18 +62,10 @@ jobs:
         # installs java 1.8, android's SDK/NDK and p4a
         - make -f ci/makefiles/osx.mk
         - export JAVA_HOME=/Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
-        # Run a background process (like we do with linux tests)
-        - while sleep 540; do echo "==== Still running (travis, don't kill me) ===="; done &
-      script:
-        - >
-          cd testapps && python3 setup_testapp_python3_sqlite_openssl.py apk
-          --sdk-dir $HOME/.android/android-sdk
-          --ndk-dir $HOME/.android/android-ndk
-          --requirements libffi,sdl2,pyjnius,kivy,python3,openssl,requests,sqlite3,setuptools
-          --arch=armeabi-v7a
+      script: make testapps/python3/armeabi-v7a PYTHON_WITH_VERSION=python3
     - <<: *testing
       name: Python 2 armeabi-v7a (with numpy)
-      env: COMMAND='. venv/bin/activate && cd testapps/ && python setup_testapp_python2_sqlite_openssl.py apk --sdk-dir $ANDROID_SDK_HOME --ndk-dir $ANDROID_NDK_HOME --requirements sdl2,pyjnius,kivy,python2,openssl,requests,sqlite3,setuptools,numpy'
+      script: make docker/run/make/testapps/python2/armeabi-v7a
     - <<: *testing
       name: Rebuild updated recipes
-      env: COMMAND='. venv/bin/activate && ./ci/rebuild_updated_recipes.py'
+      script: make docker/run/make/rebuild_updated_recipes
diff --git a/Dockerfile.py3 b/Dockerfile.py3
@@ -25,9 +25,11 @@ ENV LANG="en_US.UTF-8" \
     LANGUAGE="en_US.UTF-8" \
     LC_ALL="en_US.UTF-8"
 
-RUN apt -y update -qq \
-    && apt -y install -qq --no-install-recommends curl unzip ca-certificates \
-    && apt -y autoremove
+RUN apt -y update -qq > /dev/null && apt -y install -qq --no-install-recommends \
+	ca-certificates \
+    curl \
+    && apt -y autoremove \
+    && apt -y clean
 
 # retry helper script, refs:
 # https://github.com/kivy/python-for-android/issues/1306
@@ -37,38 +39,52 @@ RUN curl https://raw.githubusercontent.com/kadwanev/retry/1.0.1/retry \
 
 ENV USER="user"
 ENV HOME_DIR="/home/${USER}"
-ENV ANDROID_HOME="${HOME_DIR}/.android"
-ENV WORK_DIR="${HOME_DIR}" \
-    PATH="${HOME_DIR}/.local/bin:${PATH}"
+ENV WORK_DIR="${HOME_DIR}/app" \
+    PATH="${HOME_DIR}/.local/bin:${PATH}" \
+    ANDROID_HOME="${HOME_DIR}/.android" \
+    JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64"
 
-# install system dependencies
-RUN ${RETRY} apt -y install -qq --no-install-recommends \
-        python3 virtualenv python3-pip python3-venv \
-        wget lbzip2 patch sudo python python-pip \
-    && apt -y autoremove
 
-# build dependencies
-# https://buildozer.readthedocs.io/en/latest/installation.html#android-on-ubuntu-16-04-64bit
+# install system dependencies
 RUN dpkg --add-architecture i386 \
-    && ${RETRY} apt -y update -qq \
+    && ${RETRY} apt -y update -qq > /dev/null \
     && ${RETRY} apt -y install -qq --no-install-recommends \
-        build-essential ccache git python3 python3-dev \
-        libncurses5:i386 libstdc++6:i386 libgtk2.0-0:i386 \
-        libpangox-1.0-0:i386 libpangoxft-1.0-0:i386 libidn11:i386 \
-        zip zlib1g-dev zlib1g:i386 \
-    && apt -y autoremove
-
-# specific recipes dependencies (e.g. libffi requires autoreconf binary)
-RUN ${RETRY} apt -y install -qq --no-install-recommends \
-        libffi-dev autoconf automake cmake gettext libltdl-dev libtool pkg-config \
+    autoconf \
+    automake \
+    build-essential \
+    ccache \
+    cmake \
+    gettext \
+    git \
+    lbzip2 \
+    libffi-dev \
+    libgtk2.0-0:i386 \
+    libidn11:i386 \
+    libltdl-dev \
+    libncurses5:i386 \
+    libpangox-1.0-0:i386 \
+    libpangoxft-1.0-0:i386 \
+    libstdc++6:i386 \
+    libtool \
+    openjdk-8-jdk \
+    patch \
+    pkg-config \
+    python \
+    python-pip \
+    python3 \
+    python3-dev \
+    python3-pip \
+    python3-venv \
+    sudo \
+    unzip \
+    virtualenv \
+    wget \
+    zip \
+    zlib1g-dev \
+    zlib1g:i386 \
     && apt -y autoremove \
     && apt -y clean
 
-# Install Java and set JAVA_HOME (to accept android's SDK licenses)
-RUN ${RETRY} apt -y install -qq --no-install-recommends openjdk-8-jdk \
-    && apt -y autoremove && apt -y clean
-ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
-
 # prepare non root env
 RUN useradd --create-home --shell /bin/bash ${USER}
 
@@ -80,15 +96,18 @@ RUN echo "%sudo ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
 RUN pip2 install --upgrade Cython==0.28.6
 
 WORKDIR ${WORK_DIR}
-COPY --chown=user:user . ${WORK_DIR}
-RUN mkdir ${ANDROID_HOME} && chown --recursive ${USER} ${ANDROID_HOME}
+RUN mkdir ${ANDROID_HOME} && chown --recursive ${USER} ${HOME_DIR} ${ANDROID_HOME}
 USER ${USER}
 
 # Download and install android's NDK/SDK
-RUN make -f ci/makefiles/android.mk target_os=linux
+COPY ci/makefiles/android.mk /tmp/android.mk
+RUN make --file /tmp/android.mk target_os=linux \
+    && sudo rm /tmp/android.mk
 
 # install python-for-android from current branch
-RUN virtualenv --python=python3 venv \
-    && . venv/bin/activate \
-    && pip3 install --upgrade Cython==0.28.6 \
-    && pip3 install -e .
+COPY --chown=user:user Makefile README.md setup.py pythonforandroid/__init__.py ${WORK_DIR}/
+RUN mkdir pythonforandroid \
+    && mv __init__.py pythonforandroid/ \
+    && make virtualenv
+
+COPY --chown=user:user . ${WORK_DIR}
diff --git a/Makefile b/Makefile
@@ -0,0 +1,78 @@
+VIRTUAL_ENV ?= venv
+PIP=$(VIRTUAL_ENV)/bin/pip
+TOX=`which tox`
+ACTIVATE=$(VIRTUAL_ENV)/bin/activate
+PYTHON=$(VIRTUAL_ENV)/bin/python
+FLAKE8=$(VIRTUAL_ENV)/bin/flake8
+PYTEST=$(VIRTUAL_ENV)/bin/pytest
+SOURCES=src/ tests/
+PYTHON_MAJOR_VERSION=3
+PYTHON_MINOR_VERSION=6
+PYTHON_VERSION=$(PYTHON_MAJOR_VERSION).$(PYTHON_MINOR_VERSION)
+PYTHON_MAJOR_MINOR=$(PYTHON_MAJOR_VERSION)$(PYTHON_MINOR_VERSION)
+PYTHON_WITH_VERSION=python$(PYTHON_VERSION)
+DOCKER_IMAGE=kivy/python-for-android
+ANDROID_SDK_HOME ?= $(HOME)/.android/android-sdk
+ANDROID_NDK_HOME ?= $(HOME)/.android/android-ndk
+
+
+all: virtualenv
+
+$(VIRTUAL_ENV):
+	virtualenv --python=$(PYTHON_WITH_VERSION) $(VIRTUAL_ENV)
+	$(PIP) install Cython==0.28.6
+	$(PIP) install -e .
+
+virtualenv: $(VIRTUAL_ENV)
+
+# ignores test_pythonpackage.py since it runs for too long
+test:
+	$(TOX) -- tests/ --ignore tests/test_pythonpackage.py
+	@if test -n "$$CI"; then .tox/py$(PYTHON_MAJOR_MINOR)/bin/coveralls; fi; \
+
+rebuild_updated_recipes: virtualenv
+	$(PYTHON) ci/rebuild_updated_recipes.py
+
+testapps/python2/armeabi-v7a: virtualenv
+	. $(ACTIVATE) && cd testapps/ && \
+    python setup_testapp_python2_sqlite_openssl.py apk --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \
+    --requirements sdl2,pyjnius,kivy,python2,openssl,requests,sqlite3,setuptools,numpy
+
+testapps/python3/arm64-v8a: virtualenv
+	. $(ACTIVATE) && cd testapps/ && \
+    python setup_testapp_python3_sqlite_openssl.py apk --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \
+    --arch=arm64-v8a
+
+testapps/python3/armeabi-v7a: virtualenv
+	. $(ACTIVATE) && cd testapps/ && \
+    python setup_testapp_python3_sqlite_openssl.py apk --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \
+    --requirements libffi,sdl2,pyjnius,kivy,python3,openssl,requests,sqlite3,setuptools \
+    --arch=armeabi-v7a
+
+clean:
+	find . -type d -name "__pycache__" -exec rm -r {} +
+	find . -type d -name "*.egg-info" -exec rm -r {} +
+
+clean/all: clean
+	rm -rf $(VIRTUAL_ENV) .tox/
+
+docker/pull:
+	docker pull $(DOCKER_IMAGE):latest || true
+
+docker/build:
+	docker build --cache-from=$(DOCKER_IMAGE) --tag=$(DOCKER_IMAGE) --file=Dockerfile.py3 .
+
+docker/push:
+	docker push $(DOCKER_IMAGE)
+
+docker/run/test: docker/build
+	docker run --rm --env-file=.env $(DOCKER_IMAGE) 'make test'
+
+docker/run/command: docker/build
+	docker run --rm --env-file=.env $(DOCKER_IMAGE) /bin/sh -c "$(COMMAND)"
+
+docker/run/make/%: docker/build
+	docker run --rm --env-file=.env $(DOCKER_IMAGE) make $*
+
+docker/run/shell: docker/build
+	docker run --rm --env-file=.env -it $(DOCKER_IMAGE)
