ARM Mac open-source and packaged build

.github/workflows/cpp-packaging.yml

@@ -32,9 +32,11 @@ env:
   demumbleVer: "1.1.0"
   # Use SHA256 for hashing files.
   hashCommand: "sha256sum"
-  # Xcode version 12 is the version we build the SDK with.
+  # Xcode version 12.2 is the version we build the SDK with.
   # Our MacOS runners will use the version in /Applications/Xcode_${xcodeVersion}.app
-  xcodeVersion: "12"
+  xcodeVersion: "12.2"
+  # LLVM version with ARM MachO support has no version number yet.
+  llvmVer: "5f187f0afaad33013ba03454c4749d99b1362534"
 
 jobs:
   log_inputs:
@@ -91,7 +93,7 @@ jobs:
         if: runner.os == 'macOS'
         run: sudo xcode-select -s /Applications/Xcode_${{ env.xcodeVersion }}.app/Contents/Developer
 
-      - name: fetch and build binutils
+      - name: Fetch and build binutils
         run: |
           set +e
           # Retry up to 10 times because Curl has a tendency to timeout on
@@ -105,13 +107,46 @@ jobs:
 
           tar -xf binutils.tar.xz
           mv ./binutils-${{ matrix.binutils_version }} ./binutils-src
+          mkdir /tmp/binutils-output
           cd binutils-src
-          ./configure --enable-targets=all --prefix=/tmp/binutils
-          make
+          ./configure --enable-targets=all --prefix=/tmp/binutils-output
+          make -j2
           make install
           cd -
           mkdir -p packaging-tools
-          cp -af /tmp/binutils/bin/* packaging-tools
+          cp -af /tmp/binutils-output/bin/* packaging-tools
+
+      - name: Cache LLVM (Mac only)
+        if: matrix.tools_platform == 'darwin'
+        id: cache_llvm
+        uses: actions/cache@v2
+        with:
+          path: llvm-src/llvm/build/bin
+          key: llvm-bin-${{matrix.tools_platform}}-${{env.xcodeVersion}}-${{env.llvmVer}}
+
+      - name: Fetch LLVM source (Mac only)
+        # Only fetch LLVM if we don't already have binaries from the cache.
+        if: ${{ matrix.tools_platform == 'darwin' && !steps.cache_llvm.outputs.cache-hit }}
+        uses: actions/[email protected]
+        with:
+          repository: llvm/llvm-project
+          path: llvm-src
+          ref: ${{ env.llvmVer }}
+
+      - name: Build LLVM (Mac only) 
+        # Only build LLVM if we don't already have binaries from the cache.
+        if: ${{ matrix.tools_platform == 'darwin' && !steps.cache_llvm.outputs.cache-hit }}
+        run: |
+          mkdir llvm-src/llvm/build
+          cd llvm-src/llvm/build
+          cmake -DLLVM_ENABLE_PROJECTS=clang -DCMAKE_BUILD_TYPE=Release ..
+          cmake --build . -j 3 --target llvm-objcopy --target llvm-nm --target llvm-ar
+
+      - name: Package LLVM output (Mac only)
+        if: matrix.tools_platform == 'darwin'
+        run: |
+          mkdir -p packaging-tools
+          cp -af llvm-src/llvm/build/bin/* packaging-tools
 
       - name: fetch demumble
         uses: actions/[email protected]
@@ -127,7 +162,7 @@ jobs:
           cmake --build .
           python demumble_test.py
           cd -
-          mkdir -p packaging-tools-
+          mkdir -p packaging-tools
           cp -af demumble-src/demumble packaging-tools
 
       - name: archive tools
@@ -254,7 +289,7 @@ jobs:
       matrix:
         os: [windows-latest, ubuntu-latest, macos-latest]
         build_type: ["Release", "Debug"]
-        architecture: ["x64", "x86"]
+        architecture: ["x64", "x86", "arm64"]
         msvc_runtime: ["static", "dynamic"]
         linux_abi: ["legacy", "c++11"]
         python_version: [3.7]
@@ -292,6 +327,10 @@ jobs:
           msvc_runtime: "dynamic"
         - os: ubuntu-latest
           build_type: "Debug"
+        - os: ubuntu-latest
+          architecture: "arm64"
+        - os: windows-latest
+          architecture: "arm64"
 
     steps:
       - name: setup Xcode version (macos)
@@ -308,6 +347,10 @@ jobs:
           echo "VCPKG_RESPONSE_FILE=external/vcpkg_${{ matrix.vcpkg_triplet }}_response_file.txt" >> $GITHUB_ENV
           echo "MATRIX_UNIQUE_NAME=${{ matrix.os }}-${{ matrix.build_type }}-${{ matrix.architecture }}-${{ matrix.python_version }}-${{ matrix.msvc_runtime }}-${{ matrix.linux_abi }}" >> $GITHUB_ENV
           echo "SDK_NAME=${{ matrix.sdk_platform }}-${{ matrix.architecture }}-${{ matrix.build_type }}-${{ matrix.msvc_runtime }}-${{ matrix.linux_abi }}" >> $GITHUB_ENV
+          if [[ '${{ matrix.sdk_platform }}' == 'darwin' ]]; then
+            # If Mac, also hash vcpkg cache on Xcode version.
+            echo "VCPKG_EXTRA_HASH=-xcode${{env.xcodeVersion}}" >> $GITHUB_ENV
+          fi
 
       - name: Add msbuild to PATH (windows)
         if: startsWith(matrix.os, 'windows')
@@ -318,7 +361,7 @@ jobs:
         uses: actions/cache@v2
         with:
           path: external/vcpkg/installed
-          key: dev-vcpkg-${{ matrix.architecture }}-${{ matrix.vcpkg_triplet_suffix }}-${{ matrix.msvc_runtime }}-${{ matrix.linux_abi }}-${{ hashFiles(format('{0}', env.VCPKG_RESPONSE_FILE)) }}-${{ hashFiles('.git/modules/external/vcpkg/HEAD') }}
+          key: dev-vcpkg-${{ matrix.architecture }}-${{ matrix.vcpkg_triplet_suffix }}-${{ matrix.msvc_runtime }}-${{ matrix.linux_abi }}-${{ hashFiles(format('{0}', env.VCPKG_RESPONSE_FILE)) }}-${{ hashFiles('.git/modules/external/vcpkg/HEAD') }}${{env.VCPKG_EXTRA_HASH}}
 
       - name: Cache ccache files
         if: startsWith(matrix.os, 'ubuntu') || startsWith(matrix.os, 'macos')
@@ -503,15 +546,15 @@ jobs:
             variant=$(sdk-src/build_scripts/desktop/get_variant.sh "${pkg}")
             additional_flags=(${verbose_flag})
             # Several build targets require explicitly-set binutils format to be passed
-            # to package.sh (and thus, to merge_libraries).
-            if [[ "${{ matrix.sdk_platform }}" == "darwin" && "${variant}" == "arm64" ]]; then
-              # MacOS ARM
-              additional_flags+=(-f mach-o-arm64)
+            # to package.sh (and thus, to merge_libraries), or use LLVM binutils.
+            if [[ "${{ matrix.sdk_platform }}" == "darwin" ]]; then
+              # MacOS: use LLVM binutils for both X64 and ARM64
+              additional_flags+=(-L)
             elif [[ "${{ matrix.sdk_platform }}" == "windows" && "${variant}" == *"/x64/"* ]]; then
-              # Windows x64
+              # Windows x64: force input and output target format
               additional_flags+=(-f pe-x86-64,pe-bigobj-x86-64)
             elif [[ "${{ matrix.sdk_platform }}" == "windows" && "${variant}" == *"/x86/"* ]]; then
-              # Windows x86
+              # Windows x86: force input and output target format
               additional_flags+=(-f pe-i386,pe-bigobj-i386)
             fi
             sdk-src/build_scripts/desktop/package.sh -b ${pkg} -o firebase-cpp-sdk-${{ matrix.sdk_platform }}${{ matrix.suffix }}-package -p ${{ matrix.sdk_platform }} -t bin -d ${variant} -P python3 -j ${additional_flags[*]}

.github/workflows/desktop.yml

@@ -76,6 +76,14 @@ jobs:
             xcode_version: "12.4"
           - os: windows-latest
             xcode_version: "12.4"
+          # arm64 is only for macos
+          - os: ubuntu-latest
+            architecture: "arm64"
+          - os: windows-latest
+            architecture: "arm64"
+          # arm64 can't be built on Xcode 11
+          - xcode_version: "11.7"
+            architecture: "arm64"
     steps:                
       - name: Setup Xcode version (macos)
         if: runner.os == 'macOS'
@@ -114,21 +122,25 @@ jobs:
           echo "VCPKG_SUFFIX: ${{env.VCPKG_SUFFIX}}"
           echo "VCPKG_RESPONSE_FILE=external/vcpkg_custom_data/response_files/${{ matrix.architecture }}-${{ env.VCPKG_SUFFIX }}.txt" >> $GITHUB_ENV
           echo "MATRIX_UNIQUE_NAME=${{ matrix.os }}-${{ matrix.build_type }}-${{ matrix.architecture }}-${{ matrix.msvc_runtime }}" >> $GITHUB_ENV
+          if [[ '${{ matrix.sdk_platform }}' == 'darwin' ]]; then
+            # If Mac, also hash vcpkg cache on Xcode version.
+            echo "VCPKG_EXTRA_HASH=-xcode${{matrix.xcode_version}}" >> $GITHUB_ENV
+          fi
 
       - name: Cache vcpkg C++ dependencies
         id: cache_vcpkg
         uses: actions/cache@v2
         with:
           path: external/vcpkg/installed
-          key: dev-vcpkg-${{ matrix.architecture }}-${{ env.VCPKG_SUFFIX }}-${{ matrix.msvc_runtime }}-${{ hashFiles(format('{0}', env.VCPKG_RESPONSE_FILE)) }}-${{ hashFiles('.git/modules/external/vcpkg/HEAD') }}
+          key: dev-vcpkg-${{ matrix.architecture }}-${{ env.VCPKG_SUFFIX }}-${{ matrix.msvc_runtime }}-${{ hashFiles(format('{0}', env.VCPKG_RESPONSE_FILE)) }}-${{ hashFiles('.git/modules/external/vcpkg/HEAD') }}${{ env.VCPKG_EXTRA_HASH }}
 
       - name: Cache ccache files
         if: startsWith(matrix.os, 'ubuntu') || startsWith(matrix.os, 'macos')
         id: cache_ccache
         uses: actions/cache@v2
         with:
           path: ccache_dir
-          key: dev-test-ccache-${{ env.MATRIX_UNIQUE_NAME }}
+          key: dev-test-ccache-${{ env.MATRIX_UNIQUE_NAME }}${{ env.VCPKG_EXTRA_HASH }}
 
       - name: Setup python
         uses: actions/setup-python@v2
@@ -193,7 +205,7 @@ jobs:
 
       - name: Run unit tests (windows & macos)
         # TODO: Enable tests for x86 once they are all working
-        if: (startsWith(matrix.os, 'windows') || startsWith(matrix.os, 'macos')) && matrix.architecture != 'x86'
+        if: (startsWith(matrix.os, 'windows') || startsWith(matrix.os, 'macos')) && matrix.architecture != 'x86' && matrix.architecture != 'arm64'
         env:
           LANG: en_US
         run: |

CMakeLists.txt

@@ -136,6 +136,15 @@ else()
   set(DESKTOP OFF)
 endif()
 
+# For OSX, if the architectures we are building for do not match the current
+# host, treat it as cross-compiling.
+if(DESKTOP AND APPLE)
+  if(NOT "${CMAKE_OSX_ARCHITECTURES}" MATCHES "${CMAKE_HOST_SYSTEM_PROCESSOR}")
+    message("Cross-compiling ENABLED")
+    set(CMAKE_CROSSCOMPILING ON CACHE BOOL "" FORCE)
+  endif()
+endif()
+
 if(DESKTOP AND NOT MSVC AND NOT APPLE)
   # Linux-specific option.
   if (FIREBASE_LINUX_USE_CXX11_ABI)
@@ -305,6 +314,12 @@ endif()
 
 # Include Firestore's external build early to resolve conflicts on packages.
 if(FIRESTORE_USE_EXTERNAL_CMAKE_BUILD)
+  if(APPLE)
+    # On Apple, set GRPC_BAZEL_BUILD to fix an issue with __thread.
+    # Setting this define makes gRPC use pthread's thread-local storage
+    # instead of GCC's.
+    add_definitions(-DGRPC_BAZEL_BUILD=1)
+  endif()
   set(FIRESTORE_BINARY_DIR ${FIRESTORE_SOURCE_DIR}-build)
 
   if(FIREBASE_CPP_BUILD_TESTS OR FIREBASE_CPP_BUILD_STUB_TESTS)
@@ -329,7 +344,7 @@ set(FLATBUFFERS_FLATHASH OFF CACHE BOOL "")
 set(LEVELDB_BUILD_TESTS OFF CACHE BOOL "")
 
 # Disable building flatc if cross compiling
-if(IOS OR ANDROID)
+if(IOS OR ANDROID OR CMAKE_CROSSCOMPILING)
   set(FLATBUFFERS_BUILD_FLATC OFF CACHE BOOL "")
 endif()
 
@@ -367,6 +382,14 @@ if(DESKTOP)
   set(CMAKE_USE_LIBSSH2 OFF)
   set(HTTP_ONLY ON)
   set(BUILD_TESTING OFF)
+  if(APPLE AND CMAKE_CROSSCOMPILING AND NOT CMAKE_TOOLCHAIN_FILE)
+    # curl's CMake file checks for CMAKE_TOOLCHAIN_FILE rather than checking
+    # CMAKE_CROSSCOMPILING when determining whether we are cross-compiling.
+    # So if CMAKE_CROSSCOMPILING is set and we don't have a CMAKE_TOOLCHAIN_FILE,
+    # just add a null one now so curl is happy. This only happens on Mac when
+    # building for arm64/x86_64 on the opposite machine.
+    set(CMAKE_TOOLCHAIN_FILE /dev/null)
+  endif()
   add_external_library(curl)
 
   add_external_library(libuv)
@@ -402,13 +425,15 @@ if(DESKTOP)
         ${UWEBSOCKETS_SOURCE_DIR}/src/WebSocket.cpp)
     if(MSVC)
       set(websockets_additional_defines
+        ${websockets_additional_defines}
         -DWIN32_LEAN_AND_MEAN # Ensure that windows doesn't include winsock.h by
                               # default, as it can cause issues when libraries try
                               # to include winsock2.h later on in the process.
         -DUSE_LIBUV=1
       )
     elseif(APPLE)
       set(websockets_additional_defines
+        ${websockets_additional_defines}
         -DUSE_LIBUV=1
       )
     endif()
@@ -440,7 +465,7 @@ if(DESKTOP)
   if(FIRESTORE_USE_EXTERNAL_CMAKE_BUILD)
     # The Firestore build includes protobuf and nanopb already
     list(APPEND CMAKE_MODULE_PATH ${NANOPB_SOURCE_DIR}/extra)
-    find_package(Nanopb REQUIRED)
+    find_package(Nanopb)
 
     set(PROTOBUF_FOUND ON)
   else()
@@ -462,7 +487,7 @@ if(DESKTOP)
   endif()
 endif()
 
-if(DESKTOP)
+if(DESKTOP AND NOT CMAKE_CROSSCOMPILING)
   # Desktop platforms do not need to declare this dependency, as they will build
   # flatc correctly when needed.
   set(FIREBASE_FLATBUFFERS_DEPENDENCIES "")

build_scripts/desktop/finish_darwin.sh

@@ -55,7 +55,7 @@ done
 # means that the lipo step has already occurred.
 if [[ ! -d "${packagepath}/libs/${os}/universal" ]]; then
     echo "Repackaging libraries using Mac libtool..."
-    for arch in "${architectures[*]}"; do
+    for arch in ${architectures[*]}; do
 	for lib in "${packagepath}/libs/${os}/${arch}"/*.a; do
 	    pushd $(dirname ${lib}) > /dev/null
 	    libname=$(basename "${lib}")
@@ -89,7 +89,7 @@ fi
 echo "Creating frameworks..."
 rm -rf "${packagepath}/frameworks/${os}"
 mkdir -p "${packagepath}/frameworks/${os}"
-for arch in universal "${architectures[*]}"; do
+for arch in universal ${architectures[*]}; do
     for f in "${packagepath}/libs/${os}/${arch}/"*.a; do
 	library=$f
 	framework=$(basename "${library}" | sed 's|^lib||' | sed 's|\.a$||')

build_scripts/desktop/package.sh

@@ -18,6 +18,7 @@ options:
   -f, binutils format                             default: [auto-detect]
   -j, run merge_libraries jobs in parallel
   -v, enable verbose mode
+  -L, use LLVM binutils
 example:
   build_scripts/desktop/package.sh -b firebase-cpp-sdk-linux -p linux -o package_out -v x86 -j"
 }
@@ -35,6 +36,7 @@ built_sdk_tarfile=
 binutils_format=
 temp_dir=
 run_in_parallel=0
+use_llvm_binutils=0
 
 . "${root_dir}/build_scripts/packaging.conf"
 
@@ -48,7 +50,7 @@ abspath(){
     fi
 }
 
-while getopts "f:b:o:p:d:m:P:t:hjv" opt; do
+while getopts "f:b:o:p:d:m:P:t:hjLv" opt; do
     case $opt in
         f)
             binutils_format=$OPTARG
@@ -73,6 +75,9 @@ while getopts "f:b:o:p:d:m:P:t:hjv" opt; do
         v)
             verbose=1
             ;;
+        L)
+            use_llvm_binutils=1
+            ;;
         d)
             variant=$OPTARG
             ;;
@@ -228,13 +233,19 @@ readonly -a rename_namespaces=(flatbuffers flexbuffers reflection ZLib bssl uWS
 readonly rename_string=f_b_
 
 readonly demangle_cmds=${tools_path}/c++filt,${tools_path}/demumble
-readonly binutils_objcopy=${tools_path}/objcopy
-if [[ -x ${tools_path}/nm-new ]] ; then
-    readonly binutils_nm=${tools_path}/nm-new
-else
-    readonly binutils_nm=${tools_path}/nm
+if [[ ${use_llvm_binutils} -eq 1 ]]; then
+    readonly binutils_objcopy=${tools_path}/llvm-objcopy
+    readonly binutils_nm=${tools_path}/llvm-nm
+    readonly binutils_ar=${tools_path}/llvm-ar
+else    
+    readonly binutils_objcopy=${tools_path}/objcopy
+    if [[ -x ${tools_path}/nm-new ]] ; then
+        readonly binutils_nm=${tools_path}/nm-new
+    else
+	readonly binutils_nm=${tools_path}/nm
+    fi
+    readonly binutils_ar=${tools_path}/ar
 fi
-readonly binutils_ar=${tools_path}/ar
 
 cache_file=/tmp/merge_libraries_cache.$$
 # Clean up cache file after script is finished.

external/vcpkg_custom_data/response_files/arm64-osx.txt

@@ -0,0 +1,4 @@
+protobuf
+zlib
+--triplet
+arm64-osx