Optimize CI runs, by avoiding unnecessary rebuilds (#2833)

misl6 · web-flow · commit 37cb53e2629d · 2023-06-05T18:44:18.000+02:00
diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml
@@ -53,8 +53,8 @@ jobs:
         parallel: true
         flag-name: run-${{ matrix.os }}-${{ matrix.python-version }}
 
-  ubuntu_build_apk:
-    name: Unit test apk [ ${{ matrix.runs_on }} | ${{ matrix.bootstrap.name }} ]
+  ubuntu_build:
+    name: Build test APP [ ${{ matrix.runs_on }} | ${{ matrix.bootstrap.name }} ]
     needs: [flake8]
     runs-on: ${{ matrix.runs_on }}
     continue-on-error: true
@@ -68,36 +68,41 @@ jobs:
             target: testapps-with-scipy
           - name: webview
             target: testapps-webview
+          - name: service_library
+            target: testapps-service_library-aar
     steps:
     - name: Checkout python-for-android
       uses: actions/checkout@v3
-    # helps with GitHub runner getting out of space
-    - name: Free disk space
-      run: |
-        df -h
-        sudo swapoff -a
-        sudo rm -f /swapfile
-        sudo apt -y clean
-        docker rmi $(docker image ls -aq)
-        df -h
-    - name: Pull docker image
-      run: |
-        make docker/pull
-    - name: Build multi-arch apk Python 3 (armeabi-v7a, arm64-v8a, x86_64, x86)
-      run: |
-        mkdir -p apks
-        make docker/run/make/with-artifact/apk/${{ matrix.bootstrap.target }}
-    - name: Rename apk artifact to include the build platform name
-      run: |
-        mv apks/${{ env.APK_ARTIFACT_FILENAME }} apks/${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.APK_ARTIFACT_FILENAME }}
-    - name: Upload apk artifact
-      uses: actions/upload-artifact@v1
+    - name: Build python-for-android docker image
+      run: |
+        docker build --tag=kivy/python-for-android .
+    - name: Build multi-arch ${{ matrix.bootstrap.target }} artifact with docker
+      run: |
+        docker run --name p4a-latest kivy/python-for-android make ${{ matrix.bootstrap.target }}
+    - name: Copy produced artifacts from docker container (*.apk, *.aab)
+      if: matrix.bootstrap.name != 'service_library'
+      run: |
+        mkdir -p dist
+        docker cp p4a-latest:/home/user/app/testapps/on_device_unit_tests/${{ env.APK_ARTIFACT_FILENAME }} dist/
+        docker cp p4a-latest:/home/user/app/testapps/on_device_unit_tests/${{ env.AAB_ARTIFACT_FILENAME }} dist/
+    - name: Copy produced artifacts from docker container (*.aar)
+      if: matrix.bootstrap.name == 'service_library'
+      run: |
+        mkdir -p dist
+        docker cp p4a-latest:/home/user/app/testapps/on_device_unit_tests/${{ env.AAR_ARTIFACT_FILENAME }} dist/
+    - name: Rename artifacts to include the build platform name (*.apk, *.aab, *.aar)
+      run: |
+        if [ -f dist/${{ env.APK_ARTIFACT_FILENAME }} ]; then mv dist/${{ env.APK_ARTIFACT_FILENAME }} dist/${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.APK_ARTIFACT_FILENAME }}; fi
+        if [ -f dist/${{ env.AAB_ARTIFACT_FILENAME }} ]; then mv dist/${{ env.AAB_ARTIFACT_FILENAME }} dist/${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.AAB_ARTIFACT_FILENAME }}; fi
+        if [ -f dist/${{ env.AAR_ARTIFACT_FILENAME }} ]; then mv dist/${{ env.AAR_ARTIFACT_FILENAME }} dist/${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.AAR_ARTIFACT_FILENAME }}; fi
+    - name: Upload artifacts
+      uses: actions/upload-artifact@v3
       with:
-        name: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.APK_ARTIFACT_FILENAME }}
-        path: apks/${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.APK_ARTIFACT_FILENAME }}
+        name: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-artifacts
+        path: dist
 
-  macos_build_apk:
-    name: Unit test apk [ ${{ matrix.runs_on }} | ${{ matrix.bootstrap.name }} ]
+  macos_build:
+    name: Build test APP [ ${{ matrix.runs_on }} | ${{ matrix.bootstrap.name }} ]
     needs: [flake8]
     runs-on: ${{ matrix.runs_on }}
     continue-on-error: true
@@ -137,146 +142,21 @@ jobs:
           source ci/osx_ci.sh
           arm64_set_path_and_python_version 3.9.7
           make ${{ matrix.bootstrap.target }}
-      - name: Rename apk artifact to include the build platform name
-        run: |
-          mv testapps/on_device_unit_tests/${{ env.APK_ARTIFACT_FILENAME }} ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.APK_ARTIFACT_FILENAME }}
-      - name: Upload apk artifact
-        uses: actions/upload-artifact@v1
-        with:
-          name: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.APK_ARTIFACT_FILENAME }}
-          path: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.APK_ARTIFACT_FILENAME }}
-
-  ubuntu_build_aab:
-    name: Unit test aab [ ${{ matrix.runs_on }} ]
-    needs: [flake8]
-    runs-on: ${{ matrix.runs_on }}
-    continue-on-error: true
-    strategy:
-      matrix:
-        runs_on: [ubuntu-latest]
-        bootstrap:
-          - name: sdl2
-            target: testapps-with-numpy-aab
-          - name: webview
-            target: testapps-webview-aab
-    steps:
-    - name: Checkout python-for-android
-      uses: actions/checkout@v3
-    # helps with GitHub runner getting out of space
-    - name: Free disk space
-      run: |
-        df -h
-        sudo swapoff -a
-        sudo rm -f /swapfile
-        sudo apt -y clean
-        docker rmi $(docker image ls -aq)
-        df -h
-    - name: Pull docker image
-      run: |
-        make docker/pull
-    - name: Build Android App Bundle Python 3 (armeabi-v7a, arm64-v8a, x86_64, x86)
-      run: |
-        mkdir -p aabs
-        make docker/run/make/with-artifact/aab/${{ matrix.bootstrap.target }}
-    - name: Rename artifact to include the build platform name
-      run: |
-        mv aabs/${{ env.AAB_ARTIFACT_FILENAME }} aabs/${{ matrix.runs_on }}-${{ matrix.bootstrap.name}}-${{ env.AAB_ARTIFACT_FILENAME }}
-    - name: Upload apk artifact
-      uses: actions/upload-artifact@v1
-      with:
-        name: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name}}-${{ env.AAB_ARTIFACT_FILENAME }}
-        path: aabs/${{ matrix.runs_on }}-${{ matrix.bootstrap.name}}-${{ env.AAB_ARTIFACT_FILENAME }}
-
-
-  ubuntu_build_aar:
-    name: Unit test aar [ ${{ matrix.runs_on }} ]
-    needs: [flake8]
-    runs-on: ${{ matrix.runs_on }}
-    continue-on-error: true
-    strategy:
-      matrix:
-        runs_on: [ubuntu-latest]
-        bootstrap:
-          - name: service_library
-            target: testapps-service_library-aar
-    steps:
-    - name: Checkout python-for-android
-      uses: actions/checkout@v3
-    # helps with GitHub runner getting out of space
-    - name: Free disk space
-      run: |
-        df -h
-        sudo swapoff -a
-        sudo rm -f /swapfile
-        sudo apt -y clean
-        docker rmi $(docker image ls -aq)
-        df -h
-    - name: Pull docker image
-      run: |
-        make docker/pull
-    - name: Build Android AAR Python 3 (arm64-v8a)
-      run: |
-        mkdir -p aars
-        make docker/run/make/with-artifact/aar/${{ matrix.bootstrap.target }}
-    - name: Rename artifact to include the build platform name
-      run: |
-        mv aars/${{ env.AAR_ARTIFACT_FILENAME }} aars/${{ matrix.runs_on }}-${{ matrix.bootstrap.name}}-${{ env.AAR_ARTIFACT_FILENAME }}
-    - name: Upload aar artifact
-      uses: actions/upload-artifact@v1
-      with:
-        name: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name}}-${{ env.AAR_ARTIFACT_FILENAME }}
-        path: aars/${{ matrix.runs_on }}-${{ matrix.bootstrap.name}}-${{ env.AAR_ARTIFACT_FILENAME }}
-
-
-  macos_build_aab:
-    name: Unit test aab [ ${{ matrix.runs_on }} | ${{ matrix.bootstrap.name }} ]
-    needs: [flake8]
-    runs-on: ${{ matrix.runs_on }}
-    continue-on-error: true
-    strategy:
-      matrix:
-        runs_on: [macos-latest, apple-silicon-m1]
-        bootstrap:
-          - name: sdl2
-            target: testapps-with-numpy-aab
-          - name: webview
-            target: testapps-webview-aab
-    env:
-      ANDROID_HOME: ${HOME}/.android
-      ANDROID_SDK_ROOT: ${HOME}/.android/android-sdk
-      ANDROID_SDK_HOME: ${HOME}/.android/android-sdk
-      ANDROID_NDK_HOME: ${HOME}/.android/android-ndk
-    steps:
-      - name: Checkout python-for-android
-        uses: actions/checkout@v3
-      - name: Install python-for-android
-        run: |
-          source ci/osx_ci.sh
-          arm64_set_path_and_python_version 3.9.7
-          python3 -m pip install -e .
-      - name: Install prerequisites via pythonforandroid/prerequisites.py (Experimental)
-        run: |
-          source ci/osx_ci.sh
-          arm64_set_path_and_python_version 3.9.7
-          python3 pythonforandroid/prerequisites.py
-      - name: Install dependencies (Legacy)
+      - name: Copy produced artifacts into dist/ (*.apk, *.aab)
         run: |
-          source ci/osx_ci.sh
-          arm64_set_path_and_python_version 3.9.7
-          make --file ci/makefiles/osx.mk
-      - name: Build multi-arch sdl2 aab Python 3 (armeabi-v7a, arm64-v8a, x86_64, x86)
-        run: |
-          source ci/osx_ci.sh
-          arm64_set_path_and_python_version 3.9.7
-          make ${{ matrix.bootstrap.target }}
-      - name: Rename sdl2 artifact to include the build platform name
+          mkdir -p dist
+          cp testapps/on_device_unit_tests/*.apk dist/
+          cp testapps/on_device_unit_tests/*.aab dist/
+          ls -l dist/
+      - name: Rename artifacts to include the build platform name (*.apk, *.aab)
         run: |
-          mv testapps/on_device_unit_tests/${{ env.AAB_ARTIFACT_FILENAME }} ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.AAB_ARTIFACT_FILENAME }}
-      - name: Upload sdl2 apk artifact
-        uses: actions/upload-artifact@v1
+          if [ -f dist/${{ env.APK_ARTIFACT_FILENAME }} ]; then mv dist/${{ env.APK_ARTIFACT_FILENAME }} dist/${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.APK_ARTIFACT_FILENAME }}; fi
+          if [ -f dist/${{ env.AAB_ARTIFACT_FILENAME }} ]; then mv dist/${{ env.AAB_ARTIFACT_FILENAME }} dist/${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.AAB_ARTIFACT_FILENAME }}; fi
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
         with:
-          name: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.AAB_ARTIFACT_FILENAME }}
-          path: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-${{ env.AAB_ARTIFACT_FILENAME }}
+          name: ${{ matrix.runs_on }}-${{ matrix.bootstrap.name }}-artifacts
+          path: dist
 
   ubuntu_rebuild_updated_recipes:
     name: Test updated recipes for arch ${{ matrix.android_arch }} [ ubuntu-latest ]
diff --git a/Makefile b/Makefile
@@ -36,47 +36,51 @@ rebuild_updated_recipes: virtualenv
 	ANDROID_SDK_HOME=$(ANDROID_SDK_HOME) ANDROID_NDK_HOME=$(ANDROID_NDK_HOME) \
 	$(PYTHON) ci/rebuild_updated_recipes.py $(REBUILD_UPDATED_RECIPES_EXTRA_ARGS)
 
-testapps-with-numpy: virtualenv
+testapps-with-numpy: testapps-with-numpy/debug/apk testapps-with-numpy/release/aab
+
+# testapps-with-numpy/MODE/ARTIFACT
+testapps-with-numpy/%: virtualenv
+	$(eval MODE := $(word 2, $(subst /, ,$@)))
+	$(eval ARTIFACT := $(word 3, $(subst /, ,$@)))
+	@echo Building testapps-with-numpy for $(MODE) mode and $(ARTIFACT) artifact
 	. $(ACTIVATE) && cd testapps/on_device_unit_tests/ && \
-    python setup.py apk --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \
+    python setup.py $(ARTIFACT) --$(MODE) --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \
     --requirements libffi,sdl2,pyjnius,kivy,python3,openssl,requests,urllib3,chardet,idna,sqlite3,setuptools,numpy \
     --arch=armeabi-v7a --arch=arm64-v8a --arch=x86_64 --arch=x86 \
 	--permission "(name=android.permission.WRITE_EXTERNAL_STORAGE;maxSdkVersion=18)" --permission "(name=android.permission.INTERNET)"
 
-testapps-with-scipy: virtualenv
+testapps-with-scipy: testapps-with-scipy/debug/apk testapps-with-scipy/release/aab
+
+# testapps-with-scipy/MODE/ARTIFACT
+testapps-with-scipy/%: virtualenv
+	$(eval MODE := $(word 2, $(subst /, ,$@)))
+	$(eval ARTIFACT := $(word 3, $(subst /, ,$@)))
+	@echo Building testapps-with-scipy for $(MODE) mode and $(ARTIFACT) artifact
 	. $(ACTIVATE) && cd testapps/on_device_unit_tests/ && \
 	export LEGACY_NDK=$(ANDROID_NDK_HOME_LEGACY)  && \
-    python setup.py apk --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \
+    python setup.py $(ARTIFACT) --$(MODE) --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \
     --requirements python3,scipy,kivy \
     --arch=armeabi-v7a --arch=arm64-v8a
 
-testapps-with-numpy-aab: virtualenv
-	. $(ACTIVATE) && cd testapps/on_device_unit_tests/ && \
-    python setup.py aab --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \
-    --requirements libffi,sdl2,pyjnius,kivy,python3,openssl,requests,urllib3,chardet,idna,sqlite3,setuptools,numpy \
-    --arch=armeabi-v7a --arch=arm64-v8a --arch=x86_64 --arch=x86 --release \
-	--permission "(name=android.permission.WRITE_EXTERNAL_STORAGE;maxSdkVersion=18)" --permission "(name=android.permission.INTERNET)"
+testapps-webview: testapps-webview/debug/apk testapps-webview/release/aab
 
-testapps-service_library-aar: virtualenv
+# testapps-webview/MODE/ARTIFACT
+testapps-webview/%: virtualenv
+	$(eval MODE := $(word 2, $(subst /, ,$@)))
+	$(eval ARTIFACT := $(word 3, $(subst /, ,$@)))
+	@echo Building testapps-webview for $(MODE) mode and $(ARTIFACT) artifact
 	. $(ACTIVATE) && cd testapps/on_device_unit_tests/ && \
-    python setup.py aar --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \
-    --bootstrap service_library \
-    --requirements python3 \
-    --arch=arm64-v8a --arch=x86 --release
-
-testapps-webview: virtualenv
-	. $(ACTIVATE) && cd testapps/on_device_unit_tests/ && \
-    python setup.py apk --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \
+    python setup.py $(ARTIFACT) --$(MODE) --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \
     --bootstrap webview \
     --requirements sqlite3,libffi,openssl,pyjnius,flask,python3,genericndkbuild \
     --arch=armeabi-v7a --arch=arm64-v8a --arch=x86_64 --arch=x86
 
-testapps-webview-aab: virtualenv
+testapps-service_library-aar: virtualenv 
 	. $(ACTIVATE) && cd testapps/on_device_unit_tests/ && \
-    python setup.py aab --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \
-    --bootstrap webview \
-    --requirements sqlite3,libffi,openssl,pyjnius,flask,python3,genericndkbuild \
-    --arch=armeabi-v7a --arch=arm64-v8a --arch=x86_64 --arch=x86 --release
+    python setup.py aar --sdk-dir $(ANDROID_SDK_HOME) --ndk-dir $(ANDROID_NDK_HOME) \
+    --bootstrap service_library \
+    --requirements python3 \
+    --arch=arm64-v8a --arch=x86 --release
 
 testapps/%: virtualenv
 	$(eval $@_APP_ARCH := $(shell basename $*))
@@ -106,21 +110,6 @@ docker/run/test: docker/build
 docker/run/command: docker/build
 	docker run --rm --env-file=.env $(DOCKER_IMAGE) /bin/sh -c "$(COMMAND)"
 
-docker/run/make/with-artifact/apk/%: docker/build
-	docker run --name p4a-latest --env-file=.env $(DOCKER_IMAGE) make $*
-	docker cp p4a-latest:/home/user/app/testapps/on_device_unit_tests/bdist_unit_tests_app-debug-1.1.apk ./apks
-	docker rm -fv p4a-latest
-
-docker/run/make/with-artifact/aar/%: docker/build
-	docker run --name p4a-latest --env-file=.env $(DOCKER_IMAGE) make $*
-	docker cp p4a-latest:/home/user/app/testapps/on_device_unit_tests/bdist_unit_tests_app-release-1.1.aar ./aars
-	docker rm -fv p4a-latest
-
-docker/run/make/with-artifact/aab/%: docker/build
-	docker run --name p4a-latest --env-file=.env $(DOCKER_IMAGE) make $*
-	docker cp p4a-latest:/home/user/app/testapps/on_device_unit_tests/bdist_unit_tests_app-release-1.1.aab ./aabs
-	docker rm -fv p4a-latest
-
 docker/run/make/rebuild_updated_recipes: docker/build
 	docker run --name p4a-latest -e REBUILD_UPDATED_RECIPES_EXTRA_ARGS --env-file=.env $(DOCKER_IMAGE) make rebuild_updated_recipes
 
