Upgrade flatbuffers version to 23.5.26 (#465)

Summary:
`23.5.26` from conda-forge https://anaconda.org/conda-forge/flatbuffers matches the third-party commit.  This is to avoid compilation issue when the version of `flatbuffers` from conda (used by CI) is different than the third-party commit (used when compiling).

Pull Request resolved: #465

Reviewed By: cccclai

Differential Revision: D49567226

Pulled By: huydhn

fbshipit-source-id: 8926631eebf5faad3d0aaaa288fd369c18b472f7

Author: huydhn

.ci/docker/conda-env-ci.txt

@@ -1,4 +1 @@
-# TODO: We might need to update this to install flatbuffers from the pinned commit
-# in fbcode/executorch/third-party/flatbuffers.submodule.txt
-flatbuffers=2.0.0
 cmake=3.22.1

.ci/scripts/setup-linux.sh

@@ -20,5 +20,6 @@ fi
 
 # As Linux job is running inside a Docker container, all of its dependencies
 # have already been installed
+install_flatc_from_source
 install_executorch
 build_executorch_runner "${BUILD_TOOL}"

.ci/scripts/setup-macos.sh

@@ -80,5 +80,6 @@ install_buck
 install_conda
 install_pip_dependencies
 print_cmake_info
+install_flatc_from_source
 install_executorch
 build_executorch_runner "${BUILD_TOOL}"

.ci/scripts/utils.sh

@@ -27,7 +27,7 @@ install_executorch() {
 install_conda() {
   pushd .ci/docker || return
   # Install conda dependencies like flatbuffer
-  conda install --file conda-env-ci.txt
+  conda install -y --file conda-env-ci.txt
   popd || return
 }
 
@@ -49,6 +49,25 @@ install_pip_dependencies() {
   popd || return
 }
 
+install_flatc_from_source() {
+  # NB: This function could be used to install flatbuffer from source
+  pushd third-party/flatbuffers || return
+
+  cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release
+  if [ "$(uname)" == "Darwin" ]; then
+    CMAKE_JOBS=$(( $(sysctl -n hw.ncpu) - 1 ))
+  else
+    CMAKE_JOBS=$(( $(nproc) - 1 ))
+  fi
+  cmake --build . -j "${CMAKE_JOBS}"
+
+  # Copy the flatc binary to conda path
+  EXEC_PATH=$(dirname "$(which python)")
+  cp flatc "${EXEC_PATH}"
+
+  popd || return
+}
+
 build_executorch_runner_buck2() {
   # Build executorch runtime with retry as this step is flaky on macos CI
   retry buck2 build //examples/executor_runner:executor_runner

.github/workflows/_unittest.yml

@@ -15,6 +15,10 @@ on:
 jobs:
   linux:
     uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
+    strategy:
+      matrix:
+        include:
+          - build-tool: buck2
     with:
       runner: linux.2xlarge
       docker-image: ${{ inputs.docker-image }}
@@ -28,8 +32,10 @@ jobs:
         CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
         conda activate "${CONDA_ENV}"
 
-        # Just need to install executorch, everything else has been setup
-        pip install .
+        BUILD_TOOL=${{ matrix.build-tool }}
+        # Setup MacOS dependencies as there is no Docker support on MacOS atm
+        PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh "${BUILD_TOOL}"
+
         # Run pytest with coverage
         pytest -n auto --cov=./ --cov-report=xml
 
@@ -53,8 +59,6 @@ jobs:
         # Setup MacOS dependencies as there is no Docker support on MacOS atm
         PYTHON_EXECUTABLE=python bash .ci/scripts/setup-macos.sh "${BUILD_TOOL}"
 
-        # Just need to install executorch, everything else has been setup
-        pip install .
         # Run pytest with coverage
         pytest -n auto --cov=./ --cov-report=xml
         # Run gtest

.github/workflows/pull.yml

@@ -130,12 +130,12 @@ jobs:
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       script: |
         WORKSPACE=$(pwd)
-
         pushd "${WORKSPACE}/pytorch/executorch"
 
         BUILD_TOOL=${{ matrix.build-tool }}
         # Setup MacOS dependencies as there is no Docker support on MacOS atm
         PYTHON_EXECUTABLE=python bash .ci/scripts/setup-macos.sh "${BUILD_TOOL}"
+
         # Build and test custom ops
         PYTHON_EXECUTABLE=python bash examples/custom_ops/test_custom_ops.sh "${BUILD_TOOL}"
         popd
@@ -179,12 +179,12 @@ jobs:
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
       script: |
         WORKSPACE=$(pwd)
-
         pushd "${WORKSPACE}/pytorch/executorch"
 
         BUILD_TOOL=${{ matrix.build-tool }}
         # Setup MacOS dependencies as there is no Docker support on MacOS atm
         PYTHON_EXECUTABLE=python bash .ci/scripts/setup-macos.sh "${BUILD_TOOL}"
+
         # Build and test selective build
         PYTHON_EXECUTABLE=python bash examples/selective_build/test_selective_build.sh "${BUILD_TOOL}"
         popd