Resolve the conflict

Author: chuntl

.ci/docker/build.sh

@@ -37,6 +37,12 @@ case "${IMAGE_NAME}" in
     ARM_SDK=yes
     CLANG_VERSION=12
     ;;
+  executorch-ubuntu-22.04-clang12-android)
+    LINTRUNNER=""
+    CLANG_VERSION=12
+    # From https://developer.android.com/ndk/downloads
+    ANDROID_NDK_VERSION=r26c
+    ;;
   *)
     echo "Invalid image name ${IMAGE_NAME}"
     exit 1
@@ -66,6 +72,7 @@ docker build \
   --build-arg "LINTRUNNER=${LINTRUNNER:-}" \
   --build-arg "BUILD_DOCS=${BUILD_DOCS}" \
   --build-arg "ARM_SDK=${ARM_SDK:-}" \
+  --build-arg "ANDROID_NDK_VERSION=${ANDROID_NDK_VERSION:-}" \
   -f "${OS}"/Dockerfile \
   "$@" \
   .

.ci/docker/ci_commit_pins/pytorch.txt

@@ -1 +1 @@
-f5b99976adcbb01fd71bd0a39ea15bdac6c9e48a
+6ca9ae4f8693639c395544327f7e362441a58c79

.ci/docker/common/install_android.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -ex
+
+# Double check if the NDK version is set
+[ -n "${ANDROID_NDK_VERSION}" ]
+
+install_prerequiresites() {
+  apt-get update
+
+  # NB: Need OpenJDK 17 at the minimum
+  apt-get install -y --no-install-recommends \
+    openjdk-17-jdk \
+    ca-certificates-java \
+    ant
+
+  # Cleanup package manager
+  apt-get autoclean && apt-get clean
+  rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+}
+
+install_ndk() {
+  NDK_INSTALLATION_DIR=/opt/ndk
+  mkdir -p "${NDK_INSTALLATION_DIR}"
+
+  pushd /tmp
+  # The NDK installation is cached on ossci-android S3 bucket
+  curl -Os --retry 3 "https://ossci-android.s3.amazonaws.com/android-ndk-${ANDROID_NDK_VERSION}-linux.zip"
+  unzip -qo "android-ndk-${ANDROID_NDK_VERSION}-linux.zip"
+
+  # Print the content for manual verification
+  ls -lah "android-ndk-${ANDROID_NDK_VERSION}"
+  mv "android-ndk-${ANDROID_NDK_VERSION}"/* "${NDK_INSTALLATION_DIR}"
+
+  popd
+}
+
+install_cmdtools() {
+  CMDTOOLS_FILENAME=commandlinetools-linux-11076708_latest.zip
+
+  pushd /tmp
+  # The file is cached on ossci-android S3 bucket
+  curl -Os --retry 3 "https://ossci-android.s3.us-west-1.amazonaws.com/${CMDTOOLS_FILENAME}"
+  unzip -qo "${CMDTOOLS_FILENAME}" -d /opt
+
+  ls -lah /opt/cmdline-tools/bin
+  popd
+}
+
+install_sdk() {
+  SDK_INSTALLATION_DIR=/opt/android/sdk
+  mkdir -p "${SDK_INSTALLATION_DIR}"
+
+  # These are the tools needed to build Android apps
+  yes | /opt/cmdline-tools/bin/sdkmanager --sdk_root="${SDK_INSTALLATION_DIR}" --install "platforms;android-34"
+  yes | /opt/cmdline-tools/bin/sdkmanager --sdk_root="${SDK_INSTALLATION_DIR}" --install "build-tools;33.0.1"
+  # And some more tools for future emulator tests
+  yes | /opt/cmdline-tools/bin/sdkmanager --sdk_root="${SDK_INSTALLATION_DIR}" --install "platform-tools"
+  yes | /opt/cmdline-tools/bin/sdkmanager --sdk_root="${SDK_INSTALLATION_DIR}" --install "tools"
+}
+
+install_prerequiresites
+install_ndk
+install_cmdtools
+install_sdk

.ci/docker/common/install_conda.sh

@@ -44,13 +44,15 @@ install_pip_dependencies() {
 }
 
 fix_conda_ubuntu_libstdcxx() {
+  cat /etc/issue
   # WARNING: This is a HACK from PyTorch core to be able to build PyTorch on 22.04.
-  # The issue still exists with the latest conda 23.10.0-1 at the time of writing
-  # (2023/11/16).
+  # Specifically, ubuntu-20+ all comes lib libstdc++ newer than 3.30+, but anaconda
+  # is stuck with 3.29. So, remove libstdc++6.so.3.29 as installed by
+  # https://anaconda.org/anaconda/libstdcxx-ng/files?version=11.2.0
   #
   # PyTorch sev: https://github.com/pytorch/pytorch/issues/105248
   # Ref: https://github.com/pytorch/pytorch/blob/main/.ci/docker/common/install_conda.sh
-  if grep -e "[12][82].04.[623]" /etc/issue >/dev/null; then
+  if grep -e "2[02].04." /etc/issue >/dev/null; then
     rm "/opt/conda/envs/py_${PYTHON_VERSION}/lib/libstdc++.so.6"
   fi
 }

.ci/docker/ubuntu/Dockerfile

@@ -71,6 +71,12 @@ COPY ./requirements-lintrunner.txt requirements-lintrunner.txt
 RUN if [ -n "${LINTRUNNER}" ]; then bash ./install_linter.sh; fi
 RUN rm install_linter.sh utils.sh requirements-lintrunner.txt
 
+ARG ANDROID_NDK_VERSION
+# Install lintrunner if needed
+COPY ./common/install_android.sh install_android.sh
+RUN if [ -n "${ANDROID_NDK_VERSION}" ]; then bash ./install_android.sh; fi
+RUN rm install_android.sh
+
 ARG ARM_SDK
 COPY --chown=ci-user:ci-user ./arm /opt/arm
 # Set up ARM SDK if needed

.ci/scripts/setup-macos.sh

@@ -23,8 +23,8 @@ install_buck() {
     brew install zstd
   fi
 
-  if ! command -v wget &> /dev/null; then
-    brew install wget
+  if ! command -v curl &> /dev/null; then
+    brew install curl
   fi
 
   pushd .ci/docker
@@ -37,7 +37,7 @@ install_buck() {
   # --version doesn't say anything w.r.t its release version, i.e. 2024-02-15.
   # See D53878006 for more details.
   BUCK2=buck2-aarch64-apple-darwin.zst
-  wget -q "https://ossci-macos.s3.amazonaws.com/${BUCK2}"
+  curl -s "https://ossci-macos.s3.amazonaws.com/${BUCK2}" -o "${BUCK2}"
 
   zstd -d "${BUCK2}" -o buck2
 

.ci/scripts/utils.sh

@@ -19,7 +19,7 @@ install_executorch() {
   which pip
   # Install executorch, this assumes that Executorch is checked out in the
   # current directory
-  pip install . --no-build-isolation
+  pip install . --no-build-isolation -v
   # Just print out the list of packages for debugging
   pip list
 }

.github/pytorch-probot.yml

@@ -2,3 +2,5 @@
 ciflow_push_tags:
 - ciflow/nightly
 - ciflow/trunk
+- ciflow/binaries
+- ciflow/binaries/all

.github/workflows/_unittest.yml

@@ -33,8 +33,11 @@ jobs:
         conda activate "${CONDA_ENV}"
 
         BUILD_TOOL=${{ matrix.build-tool }}
+
         # Setup MacOS dependencies as there is no Docker support on MacOS atm
-        PYTHON_EXECUTABLE=python EXECUTORCH_BUILD_PYBIND=ON bash .ci/scripts/setup-linux.sh "${BUILD_TOOL}"
+        PYTHON_EXECUTABLE=python \
+        EXECUTORCH_BUILD_PYBIND=ON \
+        .ci/scripts/setup-linux.sh "${BUILD_TOOL}"
 
         # Run pytest with coverage
         pytest -n auto --cov=./ --cov-report=xml
@@ -59,8 +62,13 @@ jobs:
         BUILD_TOOL=${{ matrix.build-tool }}
 
         bash .ci/scripts/setup-conda.sh
+
         # Setup MacOS dependencies as there is no Docker support on MacOS atm
-        PYTHON_EXECUTABLE=python ${CONDA_RUN} EXECUTORCH_BUILD_PYBIND=ON bash .ci/scripts/setup-macos.sh "${BUILD_TOOL}"
+        PYTHON_EXECUTABLE=python \
+        EXECUTORCH_BUILD_PYBIND=ON \
+        CMAKE_ARGS="-DEXECUTORCH_BUILD_COREML=ON -DEXECUTORCH_BUILD_MPS=ON -DEXECUTORCH_BUILD_XNNPACK=ON" \
+        ${CONDA_RUN} --no-capture-output \
+        .ci/scripts/setup-macos.sh "${BUILD_TOOL}"
 
         # Run pytest with coverage
         ${CONDA_RUN} pytest -n auto --cov=./ --cov-report=xml

.github/workflows/android.yml

@@ -0,0 +1,47 @@
+name: Build ExecuTorch Android demo apps
+
+on:
+  push:
+    branches:
+      - main
+      - release/*
+  pull_request:
+    paths:
+      - .ci/docker/**
+      - .github/workflows/android.yml
+      - install_requirements.sh
+      - examples/demo-apps/**
+      - extension/module/**
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}-${{ github.event_name == 'schedule' }}
+  cancel-in-progress: true
+
+jobs:
+  test-demo-android:
+    name: test-demo-android
+    uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
+    strategy:
+      matrix:
+        include:
+          - build-tool: buck2
+    with:
+      # NB: The example model dl3 requires lots of memory (T161064121)
+      runner: linux.12xlarge
+      docker-image: executorch-ubuntu-22.04-clang12-android
+      submodules: 'true'
+      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+      timeout: 90
+      script: |
+        set -eux
+
+        # The generic Linux job chooses to use base env, not the one setup by the image
+        CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
+        conda activate "${CONDA_ENV}"
+
+        BUILD_TOOL=${{ matrix.build-tool }}
+        # Setup MacOS dependencies as there is no Docker support on MacOS atm
+        PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh "${BUILD_TOOL}"
+        # Build Android demo app
+        bash build/test_android_ci.sh