Merge updates from main branch into admob_2021 feature branch

.github/workflows/android.yml

@@ -12,6 +12,7 @@ on:
 
 env:
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
+  GITHUB_TOKEN: ${{ github.token }}
 
 jobs:
   prepare_matrix:
@@ -45,6 +46,13 @@ jobs:
         architecture: ${{ fromJson(needs.prepare_matrix.outputs.matrix_architecture) }}
         python_version: ${{ fromJson(needs.prepare_matrix.outputs.matrix_python_version) }}
     steps:
+      - name: Store git credentials for all git commands
+        # Forces all git commands to use authenticated https, to prevent throttling.
+        shell: bash
+        run: |
+          git config --global credential.helper 'store --file /tmp/git-credentials'
+          echo 'https://${{ github.token }}@github.com' > /tmp/git-credentials
+
       - name: Check expanded matrix config
         if: github.event.inputs.expanded_matrix == '1'
         run: |

.github/workflows/cpp-packaging.yml

@@ -32,9 +32,12 @@ env:
   demumbleVer: "1.1.0"
   # Use SHA256 for hashing files.
   hashCommand: "sha256sum"
-  # Xcode version 12 is the version we build the SDK with.
+  # Xcode version 12.2 is the version we build the SDK with.
   # Our MacOS runners will use the version in /Applications/Xcode_${xcodeVersion}.app
-  xcodeVersion: "12"
+  xcodeVersion: "12.2"
+  # LLVM version with ARM MachO support has no version number yet.
+  llvmVer: "5f187f0afaad33013ba03454c4749d99b1362534"
+  GITHUB_TOKEN: ${{ github.token }}
 
 jobs:
   log_inputs:
@@ -91,7 +94,7 @@ jobs:
         if: runner.os == 'macOS'
         run: sudo xcode-select -s /Applications/Xcode_${{ env.xcodeVersion }}.app/Contents/Developer
 
-      - name: fetch and build binutils
+      - name: Fetch and build binutils
         run: |
           set +e
           # Retry up to 10 times because Curl has a tendency to timeout on
@@ -105,13 +108,46 @@ jobs:
 
           tar -xf binutils.tar.xz
           mv ./binutils-${{ matrix.binutils_version }} ./binutils-src
+          mkdir /tmp/binutils-output
           cd binutils-src
-          ./configure --enable-targets=all --prefix=/tmp/binutils
-          make
+          ./configure --enable-targets=all --prefix=/tmp/binutils-output
+          make -j2
           make install
           cd -
           mkdir -p packaging-tools
-          cp -af /tmp/binutils/bin/* packaging-tools
+          cp -af /tmp/binutils-output/bin/* packaging-tools
+
+      - name: Cache LLVM (Mac only)
+        if: matrix.tools_platform == 'darwin'
+        id: cache_llvm
+        uses: actions/cache@v2
+        with:
+          path: llvm-src/llvm/build/bin
+          key: llvm-bin-${{matrix.tools_platform}}-${{env.xcodeVersion}}-${{env.llvmVer}}
+
+      - name: Fetch LLVM source (Mac only)
+        # Only fetch LLVM if we don't already have binaries from the cache.
+        if: ${{ matrix.tools_platform == 'darwin' && !steps.cache_llvm.outputs.cache-hit }}
+        uses: actions/[email protected]
+        with:
+          repository: llvm/llvm-project
+          path: llvm-src
+          ref: ${{ env.llvmVer }}
+
+      - name: Build LLVM (Mac only) 
+        # Only build LLVM if we don't already have binaries from the cache.
+        if: ${{ matrix.tools_platform == 'darwin' && !steps.cache_llvm.outputs.cache-hit }}
+        run: |
+          mkdir llvm-src/llvm/build
+          cd llvm-src/llvm/build
+          cmake -DLLVM_ENABLE_PROJECTS=clang -DCMAKE_BUILD_TYPE=Release ..
+          cmake --build . -j 3 --target llvm-objcopy --target llvm-nm --target llvm-ar
+
+      - name: Package LLVM output (Mac only)
+        if: matrix.tools_platform == 'darwin'
+        run: |
+          mkdir -p packaging-tools
+          cp -af llvm-src/llvm/build/bin/* packaging-tools
 
       - name: fetch demumble
         uses: actions/[email protected]
@@ -127,7 +163,7 @@ jobs:
           cmake --build .
           python demumble_test.py
           cd -
-          mkdir -p packaging-tools-
+          mkdir -p packaging-tools
           cp -af demumble-src/demumble packaging-tools
 
       - name: archive tools
@@ -147,6 +183,13 @@ jobs:
     runs-on: macos-latest
     if: ${{ github.event.inputs.downloadPublicVersion == '' && github.event.inputs.downloadPreviousRun == '' }}
     steps:
+      - name: Store git credentials for all git commands
+        # Forces all git commands to use authenticated https, to prevent throttling.
+        shell: bash
+        run: |
+          git config --global credential.helper 'store --file /tmp/git-credentials'
+          echo 'https://${{ github.token }}@github.com' > /tmp/git-credentials
+
       - name: setup Xcode version (macos)
         if: runner.os == 'macOS'
         run: sudo xcode-select -s /Applications/Xcode_${{ env.xcodeVersion }}.app/Contents/Developer
@@ -207,6 +250,13 @@ jobs:
         with:
           path: sdk-src
 
+      - name: Store git credentials for all git commands
+        # Forces all git commands to use authenticated https, to prevent throttling.
+        shell: bash
+        run: |
+          git config --global credential.helper 'store --file /tmp/git-credentials'
+          echo 'https://${{ github.token }}@github.com' > /tmp/git-credentials
+
       - name: Cache NDK
         id: cache_ndk
         uses: actions/cache@v2
@@ -254,7 +304,7 @@ jobs:
       matrix:
         os: [windows-latest, ubuntu-latest, macos-latest]
         build_type: ["Release", "Debug"]
-        architecture: ["x64", "x86"]
+        architecture: ["x64", "x86", "arm64"]
         msvc_runtime: ["static", "dynamic"]
         linux_abi: ["legacy", "c++11"]
         python_version: [3.7]
@@ -292,8 +342,19 @@ jobs:
           msvc_runtime: "dynamic"
         - os: ubuntu-latest
           build_type: "Debug"
+        - os: ubuntu-latest
+          architecture: "arm64"
+        - os: windows-latest
+          architecture: "arm64"
 
     steps:
+      - name: Store git credentials for all git commands
+        # Forces all git commands to use authenticated https, to prevent throttling.
+        shell: bash
+        run: |
+          git config --global credential.helper 'store --file /tmp/git-credentials'
+          echo 'https://${{ github.token }}@github.com' > /tmp/git-credentials
+
       - name: setup Xcode version (macos)
         if: runner.os == 'macOS'
         run: sudo xcode-select -s /Applications/Xcode_${{ env.xcodeVersion }}.app/Contents/Developer
@@ -308,6 +369,10 @@ jobs:
           echo "VCPKG_RESPONSE_FILE=external/vcpkg_${{ matrix.vcpkg_triplet }}_response_file.txt" >> $GITHUB_ENV
           echo "MATRIX_UNIQUE_NAME=${{ matrix.os }}-${{ matrix.build_type }}-${{ matrix.architecture }}-${{ matrix.python_version }}-${{ matrix.msvc_runtime }}-${{ matrix.linux_abi }}" >> $GITHUB_ENV
           echo "SDK_NAME=${{ matrix.sdk_platform }}-${{ matrix.architecture }}-${{ matrix.build_type }}-${{ matrix.msvc_runtime }}-${{ matrix.linux_abi }}" >> $GITHUB_ENV
+          if [[ '${{ matrix.sdk_platform }}' == 'darwin' ]]; then
+            # If Mac, also hash vcpkg cache on Xcode version.
+            echo "VCPKG_EXTRA_HASH=-xcode${{env.xcodeVersion}}" >> $GITHUB_ENV
+          fi
 
       - name: Add msbuild to PATH (windows)
         if: startsWith(matrix.os, 'windows')
@@ -318,7 +383,7 @@ jobs:
         uses: actions/cache@v2
         with:
           path: external/vcpkg/installed
-          key: dev-vcpkg-${{ matrix.architecture }}-${{ matrix.vcpkg_triplet_suffix }}-${{ matrix.msvc_runtime }}-${{ matrix.linux_abi }}-${{ hashFiles(format('{0}', env.VCPKG_RESPONSE_FILE)) }}-${{ hashFiles('.git/modules/external/vcpkg/HEAD') }}
+          key: dev-vcpkg-${{ matrix.architecture }}-${{ matrix.vcpkg_triplet_suffix }}-${{ matrix.msvc_runtime }}-${{ matrix.linux_abi }}-${{ hashFiles(format('{0}', env.VCPKG_RESPONSE_FILE)) }}-${{ hashFiles('.git/modules/external/vcpkg/HEAD') }}${{env.VCPKG_EXTRA_HASH}}
 
       - name: Cache ccache files
         if: startsWith(matrix.os, 'ubuntu') || startsWith(matrix.os, 'macos')
@@ -503,15 +568,15 @@ jobs:
             variant=$(sdk-src/build_scripts/desktop/get_variant.sh "${pkg}")
             additional_flags=(${verbose_flag})
             # Several build targets require explicitly-set binutils format to be passed
-            # to package.sh (and thus, to merge_libraries).
-            if [[ "${{ matrix.sdk_platform }}" == "darwin" && "${variant}" == "arm64" ]]; then
-              # MacOS ARM
-              additional_flags+=(-f mach-o-arm64)
+            # to package.sh (and thus, to merge_libraries), or use LLVM binutils.
+            if [[ "${{ matrix.sdk_platform }}" == "darwin" ]]; then
+              # MacOS: use LLVM binutils for both X64 and ARM64
+              additional_flags+=(-L)
             elif [[ "${{ matrix.sdk_platform }}" == "windows" && "${variant}" == *"/x64/"* ]]; then
-              # Windows x64
+              # Windows x64: force input and output target format
               additional_flags+=(-f pe-x86-64,pe-bigobj-x86-64)
             elif [[ "${{ matrix.sdk_platform }}" == "windows" && "${variant}" == *"/x86/"* ]]; then
-              # Windows x86
+              # Windows x86: force input and output target format
               additional_flags+=(-f pe-i386,pe-bigobj-i386)
             fi
             sdk-src/build_scripts/desktop/package.sh -b ${pkg} -o firebase-cpp-sdk-${{ matrix.sdk_platform }}${{ matrix.suffix }}-package -p ${{ matrix.sdk_platform }} -t bin -d ${variant} -P python3 -j ${additional_flags[*]}
@@ -721,8 +786,8 @@ jobs:
           USE_EXPANDED_MATRIX=0
         fi
         if [[ "${{ github.event_name }}" == "schedule" ]]; then
-          # reuse flag --test_pull_request=sdk to generate report
-          generate_report=(-p test_pull_request sdk)
+          # reuse flag --test_pull_request=nightly-packaging to generate report
+          generate_report=(-p test_pull_request nightly-packaging)
         fi
         verbose_flag=
         if [[ -n "${{ github.event.inputs.verboseBuild }}" && "${{ github.event.inputs.verboseBuild }}" -ne 0 ]]; then

.github/workflows/desktop.yml

@@ -12,6 +12,7 @@ on:
 
 env:
   CCACHE_DIR: ${{ github.workspace }}/ccache_dir
+  GITHUB_TOKEN: ${{ github.token }}
 
 jobs:
   prepare_matrix:
@@ -76,7 +77,22 @@ jobs:
             xcode_version: "12.4"
           - os: windows-latest
             xcode_version: "12.4"
+          # arm64 is only for macos
+          - os: ubuntu-latest
+            architecture: "arm64"
+          - os: windows-latest
+            architecture: "arm64"
+          # arm64 can't be built on Xcode 11
+          - xcode_version: "11.7"
+            architecture: "arm64"
     steps:                
+      - name: Store git credentials for all git commands
+        # Forces all git commands to use authenticated https, to prevent throttling.
+        shell: bash
+        run: |
+          git config --global credential.helper 'store --file /tmp/git-credentials'
+          echo 'https://${{ github.token }}@github.com' > /tmp/git-credentials
+
       - name: Setup Xcode version (macos)
         if: runner.os == 'macOS'
         run: sudo xcode-select -s /Applications/Xcode_${{ matrix.xcode_version }}.app/Contents/Developer
@@ -114,21 +130,25 @@ jobs:
           echo "VCPKG_SUFFIX: ${{env.VCPKG_SUFFIX}}"
           echo "VCPKG_RESPONSE_FILE=external/vcpkg_custom_data/response_files/${{ matrix.architecture }}-${{ env.VCPKG_SUFFIX }}.txt" >> $GITHUB_ENV
           echo "MATRIX_UNIQUE_NAME=${{ matrix.os }}-${{ matrix.build_type }}-${{ matrix.architecture }}-${{ matrix.msvc_runtime }}" >> $GITHUB_ENV
+          if [[ '${{ matrix.sdk_platform }}' == 'darwin' ]]; then
+            # If Mac, also hash vcpkg cache on Xcode version.
+            echo "VCPKG_EXTRA_HASH=-xcode${{matrix.xcode_version}}" >> $GITHUB_ENV
+          fi
 
       - name: Cache vcpkg C++ dependencies
         id: cache_vcpkg
         uses: actions/cache@v2
         with:
           path: external/vcpkg/installed
-          key: dev-vcpkg-${{ matrix.architecture }}-${{ env.VCPKG_SUFFIX }}-${{ matrix.msvc_runtime }}-${{ hashFiles(format('{0}', env.VCPKG_RESPONSE_FILE)) }}-${{ hashFiles('.git/modules/external/vcpkg/HEAD') }}
+          key: dev-vcpkg-${{ matrix.architecture }}-${{ env.VCPKG_SUFFIX }}-${{ matrix.msvc_runtime }}-${{ hashFiles(format('{0}', env.VCPKG_RESPONSE_FILE)) }}-${{ hashFiles('.git/modules/external/vcpkg/HEAD') }}${{ env.VCPKG_EXTRA_HASH }}
 
       - name: Cache ccache files
         if: startsWith(matrix.os, 'ubuntu') || startsWith(matrix.os, 'macos')
         id: cache_ccache
         uses: actions/cache@v2
         with:
           path: ccache_dir
-          key: dev-test-ccache-${{ env.MATRIX_UNIQUE_NAME }}
+          key: dev-test-ccache-${{ env.MATRIX_UNIQUE_NAME }}${{ env.VCPKG_EXTRA_HASH }}
 
       - name: Setup python
         uses: actions/setup-python@v2
@@ -193,7 +213,7 @@ jobs:
 
       - name: Run unit tests (windows & macos)
         # TODO: Enable tests for x86 once they are all working
-        if: (startsWith(matrix.os, 'windows') || startsWith(matrix.os, 'macos')) && matrix.architecture != 'x86'
+        if: (startsWith(matrix.os, 'windows') || startsWith(matrix.os, 'macos')) && matrix.architecture != 'x86' && matrix.architecture != 'arm64'
         env:
           LANG: en_US
         run: |
@@ -262,3 +282,84 @@ jobs:
           path: |
             ~/Library/Logs/DiagnosticReports/*
             /tmp/bins/*
+
+  local_integration_tests: 
+    ### This job run firestore integration test on Linux Dekstop as required by firestore team. 
+    ### It's possible to add more apps here once integration tests are less flaky.
+    name: local-integration-tests-linux-openssl
+    needs: prepare_matrix
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{needs.check_and_prepare.outputs.github_ref}}
+          submodules: true
+      - name: Set env vars
+        run: |
+          echo "VCPKG_TRIPLET=x64-linux" >> $GITHUB_ENV
+          echo "VCPKG_RESPONSE_FILE=external/vcpkg_${{ env.VCPKG_TRIPLET }}_response_file.txt" >> $GITHUB_ENV
+      - name: Cache vcpkg C++ dependencies
+        id: cache_vcpkg
+        uses: actions/cache@v2
+        with:
+          path: external/vcpkg/installed
+          key: dev-vcpkg-${{ env.VCPKG_TRIPLET }}-${{ hashFiles(format('{0}', env.VCPKG_RESPONSE_FILE)) }}-${{ hashFiles('.git/modules/external/vcpkg/HEAD') }}
+      - name: Cache ccache files
+        uses: actions/cache@v2
+        with:
+          path: ccache_dir
+          key: local-integration-tests-linux-openssl-ccache
+      - name: Setup python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ fromJson(needs.prepare_matrix.outputs.matrix_python_version)[0] }}
+      - name: Prepare for integration tests
+        run: |
+          pip install -r scripts/gha/requirements.txt
+      - name: Install OpenSSL
+        run: |
+          sudo apt install openssl
+      - name: Build integration tests
+        shell: bash
+        env: 
+          CCACHE_DIR: ${{ github.workspace }}/ccache_dir
+        run: |
+          apis=$( python scripts/gha/print_matrix_configuration.py -c -w integration_tests -k apis -m=1 )
+          python scripts/gha/build_testapps.py --p Desktop \
+            --t ${apis} \
+            --output_directory "${{ github.workspace }}" \
+            --artifact_name "desktop-ubuntu-latest-openssl" \
+            --noadd_timestamp \
+            --short_output_paths \
+            --cmake_flag=-DFIREBASE_QUICK_TEST=ON
+      - name: Summarize build results
+        if: ${{ !cancelled() }}
+        shell: bash
+        run: |
+          cat build-results-desktop-ubuntu-latest-openssl.log
+          if [[ "${{ job.status }}" != "success" ]]; then
+            exit 1
+          fi
+      - name: Set up Node (12)
+        uses: actions/setup-node@v2
+        with:
+          node-version: 12.x
+      - name: Setup Firestore Emulator
+        run: |
+          npm install -g firebase-tools
+      - name: Run Desktop integration tests
+        run: |
+          cp scripts/gha/integration_testing/google-services.json testapps-desktop-ubuntu-latest-openssl/firestore/google-services.json
+          firebase emulators:exec --only firestore --project demo-example 'python scripts/gha/desktop_tester.py --testapp_dir testapps-desktop-ubuntu-latest-openssl --logfile_name "desktop-ubuntu-latest-openssl"'
+        env:
+          USE_FIRESTORE_EMULATOR: true
+      - name: Summarize test results
+        if: ${{ !cancelled() }}
+        shell: bash
+        run: |
+          cat testapps-desktop-ubuntu-latest-openssl/test-results-desktop-ubuntu-latest-openssl.log
+          if [[ "${{ job.status }}" != "success" ]]; then
+            exit 1
+          fi