New pip build system

Summary:
Compatible with `pythom -m build` instead of using the deprecated
`python setup.py bdist_wheel`.

A new custom build command builds all CMake targets, and a new custom
build_ext command installs built files into the wheel.

A new custom build_py command copies platform-independent files into the
wheel instead of copying them into the source tree.

Puts all generated files under pip-out instead of mixing with files in
the tree.

Uses Command methods like `mkpath` and `copy_file` to benefit from
setuptools concepts like dry_run, and to get logging for free.

Adds `flatc` to the wheel, but doesn't yet make it available on the
path.

Test Plan:
Iteration command to test wheel building without completely rebuilding
the system:
```
rm -rf \
    /tmp/wheel-out \
    pip-out/lib* \
    pip-out/bdist.* \
    pip-out/temp.*/cmake-out/CMake* \
    executorch.egg-info \
    third-party/flatcc/{bin,lib} \
    ; \
  python -m build \
    --wheel --outdir /tmp/wheel-out --no-isolation
```
To fully rebuild the system, also `rm -rf pip-out`.

The wheel file is written to (e.g.)
```
/tmp/wheel-out/executorch-0.1.0-cp310-cp310-linux_x86_64.whl
```

Examine it with `unzip -l`, or install it with `pip install
/tmp/wheel-out/executorch*.whl`.

Author: dbort

.gitignore

@@ -5,12 +5,8 @@ cmake-android-out/
 cmake-ios-out/
 ethos-u-scratch/
 executorch.egg-info
+pip-out/
 __pycache__/
-build/lib/
-exir/_serialize/scalar_type.fbs
-exir/_serialize/program.fbs
-sdk/bundled_program/serialize/bundled_program_schema.fbs
-sdk/bundled_program/serialize/scalar_type.fbs
 
 # Any exported models and profiling outputs
 *.pte

build/pip_data_bin_init.py.in

@@ -0,0 +1,44 @@
+# This file should be written to the wheel package as
+# `executorch/data/bin/__init__.py`.
+#
+# Setuptools will expect to be able to say something like `from
+# executorch.data.bin import mybin; mybin()` for each entry listed in the
+# [project.scripts] section of pyproject.toml. This file makes the `mybin()`
+# function execute the binary at `executorch/data/bin/mybin` and exit with that
+# binary's exit status.
+
+import subprocess
+import os
+import sys
+import types
+
+# This file should live in the target `bin` directory.
+_bin_dir = os.path.join(os.path.dirname(__file__))
+
+def _find_executable_files_under(dir):
+    """Lists all executable files in the given directory."""
+    bin_names = []
+    for filename in os.listdir(dir):
+        filepath = os.path.join(dir, filename)
+        if os.path.isfile(filepath) and os.access(filepath, os.X_OK):
+            bin_names.append(filename)
+    return bin_names
+
+# The list of binaries to create wrapper functions for. An executable should
+# exist for each name under `data/bin/<bin-name>`.
+# @nocommit: see if we can just scrape this from the filesystem to avoid the template
+_bin_names = _find_executable_files_under(_bin_dir)
+
+# We'll define functions named after each binary. Make them importable.
+__all__ = _bin_names
+
+def _run(name):
+    """Runs the named binary, which should live under _bin_dir.
+
+    Exits the current process with the return code of the subprocess.
+    """
+    raise SystemExit(subprocess.call([os.path.join(_bin_dir, name)] + sys.argv[1:], close_fds=False))
+
+# Define a function named after each of the binaries.
+for bin_name in _bin_names:
+    exec(f"def {bin_name}(): _run('{bin_name}')")

extension/aot_util/CMakeLists.txt

@@ -63,6 +63,6 @@ find_library(TORCH_PYTHON_LIBRARY torch_python
 unset(CMAKE_CXX_FLAGS_RELEASE)
 
 list(TRANSFORM _extension_aot_util__srcs PREPEND "${EXECUTORCH_ROOT}/")
-add_library(aot_util ${_extension_aot_util__srcs})
+add_library(aot_util SHARED ${_extension_aot_util__srcs})
 target_include_directories(aot_util PUBLIC ${TORCH_INCLUDE_DIRS})
 target_link_libraries(aot_util torch)

pyproject.toml

@@ -24,11 +24,25 @@ dependencies=[
   "zstd",
 ]
 
+# Tell setuptools to generate commandline wrappers for tools that we install
+# under data/bin in the pip package. This will put these commands on the user's
+# path.
+[project.scripts]
+flatc = "executorch.data.bin:flatc"
+
 [tool.setuptools.package-data]
-"*" = ["*.fbs", "*.yaml"]
+# TODO(dbort): Prune /test[s]/ dirs, /third-party/ dirs, yaml files that we
+# don't need.
+"*" = [
+  # Some backends like XNNPACK need their .fbs files.
+  "*.fbs",
+  # Some kernel libraries need their .yaml files.
+  "*.yaml",
+]
 
 [tool.setuptools.exclude-package-data]
 "*" = ["*.pyc"]
 
 [tool.usort]
+# Do not try to put "first-party" imports in their own section.
 first_party_detection = false

sdk/CMakeLists.txt

@@ -75,6 +75,8 @@ ExternalProject_Add(
   SOURCE_DIR ${CMAKE_SOURCE_DIR}/third-party/flatcc
   BINARY_DIR ${CMAKE_BINARY_DIR}/_host_build
   CMAKE_CACHE_ARGS -DFLATCC_TEST:BOOL=OFF -DFLATCC_REFLECTION:BOOL=OFF
+      # See above comment about POSITION_INDEPENDENT_CODE.
+      -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
   INSTALL_COMMAND "" # Prevent the install step, modify as needed
 )