New pip build system

Summary:
Compatible with `pythom -m build` instead of using the deprecated
`python setup.py bdist_wheel`.

A new custom build command builds all necessary CMake targets, and a new
custom build_ext command installs built files into the wheel.

A new custom build_py command copies platform-independent files into the
wheel instead of copying them into the source tree.

Puts all generated files under pip-out instead of mixing with files in
the tree.

Uses Command methods like `mkpath` and `copy_file` to benefit from
setuptools concepts like dry_run, and to get logging for free.

Adds `flatc` to the wheel and installs a wrapper to make it available
on the path.

Test Plan:
Iteration command to test wheel building without completely rebuilding
the system:
```
rm -rf \
    /tmp/wheel-out \
    pip-out/lib* \
    pip-out/bdist.* \
    pip-out/temp.*/cmake-out/CMake* \
    executorch.egg-info \
    third-party/flatcc/{bin,lib} \
    ; \
  python setup.py bdist_wheel --dist-dir /tmp/wheel-out
```
To fully rebuild the system, also `rm -rf pip-out`.

The wheel file is written to (e.g.)
```
/tmp/wheel-out/executorch-0.1.0-cp310-cp310-linux_x86_64.whl
```

Examine it with `unzip -l`, or install it with `pip install
/tmp/wheel-out/executorch*.whl`.

Author: dbort

.gitignore

@@ -5,12 +5,8 @@ cmake-android-out/
 cmake-ios-out/
 ethos-u-scratch/
 executorch.egg-info
+pip-out/
 __pycache__/
-build/lib/
-exir/_serialize/scalar_type.fbs
-exir/_serialize/program.fbs
-sdk/bundled_program/serialize/bundled_program_schema.fbs
-sdk/bundled_program/serialize/scalar_type.fbs
 
 # Any exported models and profiling outputs
 *.pte

build/pip_data_bin_init.py.in

@@ -0,0 +1,42 @@
+# This file should be written to the wheel package as
+# `executorch/data/bin/__init__.py`.
+#
+# Setuptools will expect to be able to say something like `from
+# executorch.data.bin import mybin; mybin()` for each entry listed in the
+# [project.scripts] section of pyproject.toml. This file makes the `mybin()`
+# function execute the binary at `executorch/data/bin/mybin` and exit with that
+# binary's exit status.
+
+import subprocess
+import os
+import sys
+import types
+
+# This file should live in the target `bin` directory.
+_bin_dir = os.path.join(os.path.dirname(__file__))
+
+def _find_executable_files_under(dir):
+    """Lists all executable files in the given directory."""
+    bin_names = []
+    for filename in os.listdir(dir):
+        filepath = os.path.join(dir, filename)
+        if os.path.isfile(filepath) and os.access(filepath, os.X_OK):
+            bin_names.append(filename)
+    return bin_names
+
+# The list of binaries to create wrapper functions for.
+_bin_names = _find_executable_files_under(_bin_dir)
+
+# We'll define functions named after each binary. Make them importable.
+__all__ = _bin_names
+
+def _run(name):
+    """Runs the named binary, which should live under _bin_dir.
+
+    Exits the current process with the return code of the subprocess.
+    """
+    raise SystemExit(subprocess.call([os.path.join(_bin_dir, name)] + sys.argv[1:], close_fds=False))
+
+# Define a function named after each of the binaries.
+for bin_name in _bin_names:
+    exec(f"def {bin_name}(): _run('{bin_name}')")

install_requirements.sh

@@ -41,6 +41,7 @@ for arg in "$@"; do
   esac
 done
 
+
 # Install pytorch dependencies
 #
 # Note:
@@ -69,11 +70,15 @@ pip install --force-reinstall --pre transformers==${TRANSFORMERS_VERSION}
 TORCHSR_VERSION=1.0.4
 pip install --pre torchsr==${TORCHSR_VERSION}
 
-# Install ExecuTorch after dependencies are installed.
+
+#
+# Install executorch pip package. This also makes `flatc` available on the path.
+#
+
+# Deps needed by pip install and cmake.
+# TODO(dbort): Remove tomli once min python version is 3.11.
+pip install zstd tomli setuptools wheel
+
 EXECUTORCH_BUILD_PYBIND="$EXECUTORCH_BUILD_PYBIND" \
 CMAKE_ARGS="$CMAKE_ARGS" \
-CMAKE_BUILD_PARALLEL_LEVEL=9 \
 pip install . --no-build-isolation -v
-
-# Install flatc dependency
-bash build/install_flatc.sh

pyproject.toml

@@ -20,15 +20,27 @@ dependencies=[
   "ruamel.yaml",
   "sympy",
   "tabulate",
-  "tomli",
-  "zstd",
 ]
 
+# Tell setuptools to generate commandline wrappers for tools that we install
+# under data/bin in the pip package. This will put these commands on the user's
+# path.
+[project.scripts]
+flatc = "executorch.data.bin:flatc"
+
 [tool.setuptools.package-data]
-"*" = ["*.fbs", "*.yaml"]
+# TODO(dbort): Prune /test[s]/ dirs, /third-party/ dirs, yaml files that we
+# don't need.
+"*" = [
+  # Some backends like XNNPACK need their .fbs files.
+  "*.fbs",
+  # Some kernel libraries need their .yaml files.
+  "*.yaml",
+]
 
 [tool.setuptools.exclude-package-data]
 "*" = ["*.pyc"]
 
 [tool.usort]
+# Do not try to put "first-party" imports in their own section.
 first_party_detection = false

sdk/CMakeLists.txt

@@ -77,6 +77,8 @@ ExternalProject_Add(
   SOURCE_DIR ${CMAKE_SOURCE_DIR}/third-party/flatcc
   BINARY_DIR ${CMAKE_BINARY_DIR}/_host_build
   CMAKE_CACHE_ARGS -DFLATCC_TEST:BOOL=OFF -DFLATCC_REFLECTION:BOOL=OFF
+      # See above comment about POSITION_INDEPENDENT_CODE.
+      -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
   INSTALL_COMMAND "" # Prevent the install step, modify as needed
 )