New pip build system

Summary:
Compatible with `pythom -m build` instead of using the deprecated
`python setup.py bdist_wheel`.

A new custom build command builds all necessary CMake targets, and a new
custom build_ext command installs built files into the wheel.

A new custom build_py command copies platform-independent files into the
wheel instead of copying them into the source tree.

Puts all generated files under pip-out instead of mixing with files in
the tree.

Uses Command methods like `mkpath` and `copy_file` to benefit from
setuptools concepts like dry_run, and to get logging for free.

Adds `flatc` to the wheel and installs a wrapper to make it available
on the path.

Test Plan:
Iteration command to test wheel building without completely rebuilding
the system:
```
rm -rf \
    /tmp/wheel-out \
    pip-out/lib* \
    pip-out/bdist.* \
    pip-out/temp.*/cmake-out/CMake* \
    executorch.egg-info \
    third-party/flatcc/{bin,lib} \
    ; \
  python setup.py bdist_wheel --dist-dir /tmp/wheel-out
```
To fully rebuild the system, also `rm -rf pip-out`.

The wheel file is written to (e.g.)
```
/tmp/wheel-out/executorch-0.1.0-cp310-cp310-linux_x86_64.whl
```

Examine it with `unzip -l`, or install it with `pip install
/tmp/wheel-out/executorch*.whl`.

Author: dbort

.gitignore

@@ -5,12 +5,8 @@ cmake-android-out/
 cmake-ios-out/
 ethos-u-scratch/
 executorch.egg-info
+pip-out/
 __pycache__/
-build/lib/
-exir/_serialize/scalar_type.fbs
-exir/_serialize/program.fbs
-sdk/bundled_program/serialize/bundled_program_schema.fbs
-sdk/bundled_program/serialize/scalar_type.fbs
 
 # Any exported models and profiling outputs
 *.pte

build/pip_data_bin_init.py.in

@@ -0,0 +1,42 @@
+# This file should be written to the wheel package as
+# `executorch/data/bin/__init__.py`.
+#
+# Setuptools will expect to be able to say something like `from
+# executorch.data.bin import mybin; mybin()` for each entry listed in the
+# [project.scripts] section of pyproject.toml. This file makes the `mybin()`
+# function execute the binary at `executorch/data/bin/mybin` and exit with that
+# binary's exit status.
+
+import subprocess
+import os
+import sys
+import types
+
+# This file should live in the target `bin` directory.
+_bin_dir = os.path.join(os.path.dirname(__file__))
+
+def _find_executable_files_under(dir):
+    """Lists all executable files in the given directory."""
+    bin_names = []
+    for filename in os.listdir(dir):
+        filepath = os.path.join(dir, filename)
+        if os.path.isfile(filepath) and os.access(filepath, os.X_OK):
+            bin_names.append(filename)
+    return bin_names
+
+# The list of binaries to create wrapper functions for.
+_bin_names = _find_executable_files_under(_bin_dir)
+
+# We'll define functions named after each binary. Make them importable.
+__all__ = _bin_names
+
+def _run(name):
+    """Runs the named binary, which should live under _bin_dir.
+
+    Exits the current process with the return code of the subprocess.
+    """
+    raise SystemExit(subprocess.call([os.path.join(_bin_dir, name)] + sys.argv[1:], close_fds=False))
+
+# Define a function named after each of the binaries.
+for bin_name in _bin_names:
+    exec(f"def {bin_name}(): _run('{bin_name}')")

docs/source/getting-started-setup.md

@@ -111,37 +111,12 @@ Follow these steps:
    source .executorch/bin/activate
    ```
 
-1. Install [Cmake](https://cmake.org/download)
-
-   ```bash
-   conda install cmake
-   ```
-
-   Alternatively:
-
-   ```bash
-   pip install cmake
-   ```
-
 1. Install ExecuTorch and dependencies:
 
    ```bash
    ./install_requirements.sh
    ```
 
-   **Optional:** Install ExecuTorch as an editable installation:
-   ```bash
-   pip install --editable . --config-settings editable_mode=strict --no-build-isolation
-   ```
-
-1. Expose FlatBuffers compiler:
-
-   ExecuTorch uses `flatc` to export models and builds it from sources at
-   `third-party/flatbuffers`. Make it available by adding it to the `$PATH` environment variable,
-   as prompted by the previous step, or exporting as `$FLATC_EXECUTABLE`
-   enironment variable.
-   Run `./build/install_flatc.sh` to make sure `flatc` is installed correctly.
-
 You have successfully set up your environment to work with ExecuTorch. The next
 step is to generate a sample ExecuTorch program.
 

install_requirements.sh

@@ -28,7 +28,8 @@ for arg in "$@"; do
       ;;
     coreml|mps|xnnpack)
       if [[ "$EXECUTORCH_BUILD_PYBIND" == "ON" ]]; then
-        CMAKE_ARGS="$CMAKE_ARGS -DEXECUTORCH_BUILD_${arg^^}=ON"
+        arg_upper="$(echo "${arg}" | tr '[:lower:]' '[:upper:]')"
+        CMAKE_ARGS="$CMAKE_ARGS -DEXECUTORCH_BUILD_${arg_upper}=ON"
       else
         echo "Error: $arg must follow --pybind"
         exit 1
@@ -65,6 +66,7 @@ EXIR_REQUIREMENTS=(
 
 # pip packages needed for development.
 DEVEL_REQUIREMENTS=(
+  cmake  # For building binary targets.
   setuptools  # For building the pip package.
   tomli  # Imported by extract_sources.py when using python < 3.11.
   wheel  # For building the pip package archive.
@@ -94,13 +96,9 @@ pip install --extra-index-url "${TORCH_NIGHTLY_URL}" \
     "${REQUIREMENTS_TO_INSTALL[@]}"
 
 #
-# Install executorch pip package.
+# Install executorch pip package. This also makes `flatc` available on the path.
 #
 
-EXECUTORCH_BUILD_PYBIND="$EXECUTORCH_BUILD_PYBIND" \
-CMAKE_ARGS="$CMAKE_ARGS" \
-CMAKE_BUILD_PARALLEL_LEVEL=9 \
-pip install . --no-build-isolation -v
-
-# Install flatc dependency
-bash build/install_flatc.sh
+EXECUTORCH_BUILD_PYBIND="${EXECUTORCH_BUILD_PYBIND}" \
+    CMAKE_ARGS="${CMAKE_ARGS}" \
+    pip install . --no-build-isolation -v

pyproject.toml

@@ -20,15 +20,27 @@ dependencies=[
   "ruamel.yaml",
   "sympy",
   "tabulate",
-  "tomli",
-  "zstd",
 ]
 
+# Tell setuptools to generate commandline wrappers for tools that we install
+# under data/bin in the pip package. This will put these commands on the user's
+# path.
+[project.scripts]
+flatc = "executorch.data.bin:flatc"
+
 [tool.setuptools.package-data]
-"*" = ["*.fbs", "*.yaml"]
+# TODO(dbort): Prune /test[s]/ dirs, /third-party/ dirs, yaml files that we
+# don't need.
+"*" = [
+  # Some backends like XNNPACK need their .fbs files.
+  "*.fbs",
+  # Some kernel libraries need their .yaml files.
+  "*.yaml",
+]
 
 [tool.setuptools.exclude-package-data]
 "*" = ["*.pyc"]
 
 [tool.usort]
+# Do not try to put "first-party" imports in their own section.
 first_party_detection = false

sdk/CMakeLists.txt

@@ -77,6 +77,8 @@ ExternalProject_Add(
   SOURCE_DIR ${CMAKE_SOURCE_DIR}/third-party/flatcc
   BINARY_DIR ${CMAKE_BINARY_DIR}/_host_build
   CMAKE_CACHE_ARGS -DFLATCC_TEST:BOOL=OFF -DFLATCC_REFLECTION:BOOL=OFF
+      # See above comment about POSITION_INDEPENDENT_CODE.
+      -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
   INSTALL_COMMAND "" # Prevent the install step, modify as needed
 )