New pip build system

Summary:
Compatible with `pythom -m build` instead of using the deprecated
`python setup.py bdist_wheel`.

A new custom build command builds all CMake targets, and a new custom
build_ext command installs built files into the wheel.

A new custom build_py command copies platform-independent files into the
wheel instead of copying them into the source tree.

Puts all generated files under pip-out instead of mixing with files in
the tree.

Uses Command methods like `mkpath` and `copy_file` to benefit from
setuptools concepts like dry_run, and to get logging for free.

Adds `flatc` to the wheel, but doesn't yet make it available on the
path.

Test Plan:
Iteration command to test wheel building without completely rebuilding
the system:
```
rm -rf \
    /tmp/wheel-out \
    pip-out/lib* \
    pip-out/bdist.* \
    pip-out/temp.*/cmake-out/CMake* \
    executorch.egg-info \
    third-party/flatcc/{bin,lib} \
    ; \
  python -m build \
    --wheel --outdir /tmp/wheel-out --no-isolation
```
To fully rebuild the system, also `rm -rf pip-out`.

The wheel file is written to (e.g.)
```
/tmp/wheel-out/executorch-0.1.0-cp310-cp310-linux_x86_64.whl
```

Examine it with `unzip -l`, or install it with `pip install
/tmp/wheel-out/executorch*.whl`.

Reviewers:

Subscribers:

Tasks:

Tags:

Author: dbort

.gitignore

@@ -3,13 +3,9 @@ buck-out/
 cmake-out/
 cmake-android-out/
 cmake-ios-out/
+pip-out/
 executorch.egg-info
 __pycache__/
-build/lib/
-exir/_serialize/scalar_type.fbs
-exir/_serialize/program.fbs
-sdk/bundled_program/serialize/bundled_program_schema.fbs
-sdk/bundled_program/serialize/scalar_type.fbs
 
 # Any exported models and profiling outputs
 *.pte

CMakeLists.txt

@@ -236,11 +236,11 @@ if(EXECUTORCH_BUILD_FLATC)
     )
   endif()
   set(FLATC_EXECUTABLE flatc)
-  option(FLATBUFFERS_BUILD_FLATC "" ON)
-  option(FLATBUFFERS_BUILD_FLATHASH "" OFF)
-  option(FLATBUFFERS_BUILD_FLATLIB "" OFF)
-  option(FLATBUFFERS_BUILD_TESTS "" OFF)
-  option(FLATBUFFERS_INSTALL "" OFF)
+  set(FLATBUFFERS_BUILD_FLATC ON CACHE BOOL "")
+  set(FLATBUFFERS_BUILD_FLATHASH OFF CACHE BOOL "")
+  set(FLATBUFFERS_BUILD_FLATLIB OFF CACHE BOOL "")
+  set(FLATBUFFERS_BUILD_TESTS OFF CACHE BOOL "")
+  set(FLATBUFFERS_INSTALL OFF CACHE BOOL "")
   add_subdirectory(third-party/flatbuffers)
 endif()
 if(NOT FLATC_EXECUTABLE)
@@ -301,10 +301,10 @@ endif()
 # ${CMAKE_INSTALL_PREFIX}/
 install(
   TARGETS executorch
-  DESTINATION lib
+  LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
   INCLUDES
   DESTINATION ${_common_include_directories})
-install(FILES build/executorch-config.cmake DESTINATION lib/cmake/ExecuTorch)
+install(FILES build/executorch-config.cmake DESTINATION share/cmake/executorch)
 
 #
 # executor_runner: Host tool that demonstrates program execution.
@@ -460,9 +460,6 @@ if(EXECUTORCH_BUILD_PYBIND)
       ${PYBIND_LINK_MPS}
       ${PYBIND_LINK_XNNPACK}
   )
-
-  install(TARGETS portable_lib
-          LIBRARY DESTINATION executorch/extension/pybindings)
 endif()
 
 # Print all summary

backends/apple/mps/CMakeLists.txt

@@ -77,6 +77,6 @@ target_link_libraries(mpsdelegate
 
 install(
   TARGETS mpsdelegate
-  DESTINATION lib
+  DESTINATION ${CMAKE_INSTALL_LIBDIR}
   INCLUDES
   DESTINATION ${_common_include_directories})

backends/qualcomm/CMakeLists.txt

@@ -268,7 +268,7 @@ add_subdirectory(
     ${QNN_EXECUTORCH_ROOT_DIR}/aot/ir
     ${CMAKE_CURRENT_BINARY_DIR}/qnn_executorch/ir
 )
-install(TARGETS qnn_executorch_backend DESTINATION lib)
+install(TARGETS qnn_executorch_backend DESTINATION ${CMAKE_INSTALL_LIBDIR})
 
 # QNN pybind
 if(${CMAKE_SYSTEM_PROCESSOR} MATCHES "x86_64")

extension/data_loader/CMakeLists.txt

@@ -25,6 +25,6 @@ target_compile_options(extension_data_loader PUBLIC ${_common_compile_options})
 # Install libraries
 install(
   TARGETS extension_data_loader
-  DESTINATION ${CMAKE_BINARY_DIR}/lib
+  DESTINATION ${CMAKE_INSTALL_LIBDIR}
   INCLUDES
   DESTINATION ${_common_include_directories})

extension/module/CMakeLists.txt

@@ -32,6 +32,6 @@ target_compile_options(extension_module PUBLIC -Wno-deprecated-declarations
 # Install libraries
 install(
   TARGETS extension_module
-  DESTINATION lib
+  DESTINATION ${CMAKE_INSTALL_LIBDIR}
   INCLUDES
   DESTINATION ${_common_include_directories})

extension/runner_util/CMakeLists.txt

@@ -27,6 +27,6 @@ target_compile_options(extension_runner_util PUBLIC ${_common_compile_options})
 # Install libraries
 install(
   TARGETS extension_runner_util
-  DESTINATION ${CMAKE_BINARY_DIR}/lib
+  DESTINATION ${CMAKE_INSTALL_LIBDIR}
   INCLUDES
   DESTINATION ${_common_include_directories})

install_requirements.sh

@@ -69,6 +69,8 @@ pip install --force-reinstall --pre transformers==${TRANSFORMERS_VERSION}
 TORCHSR_VERSION=1.0.4
 pip install --pre torchsr==${TORCHSR_VERSION}
 
+echo STOP # @nocommit
+exit 1
 # Install ExecuTorch after dependencies are installed.
 EXECUTORCH_BUILD_PYBIND="$EXECUTORCH_BUILD_PYBIND" \
 CMAKE_ARGS="$CMAKE_ARGS" \

kernels/portable/CMakeLists.txt

@@ -60,4 +60,5 @@ target_compile_options(portable_kernels PUBLIC ${_common_compile_options})
 # runtime
 gen_operators_lib("portable_ops_lib" portable_kernels executorch)
 
-install(TARGETS portable_kernels portable_ops_lib DESTINATION lib)
+install(TARGETS portable_kernels portable_ops_lib
+        DESTINATION ${CMAKE_INSTALL_LIBDIR})

pyproject.toml

@@ -1,5 +1,13 @@
 [build-system]
-requires = ["setuptools", "wheel"]
+requires = [
+  "setuptools",
+  "wheel",
+# @nocommit: insted we're using --no-isolation
+#  "tomli", # Needed by extract_sources.py
+#  # @nocommit: this will be the wrong torch; it's not the nightly
+#  "torch", # Provides TorchConfig.cmake
+#  "ruamel.yaml", # Needed by gen_oplist.py
+]
 build-backend = "setuptools.build_meta"
 
 [project]
@@ -26,10 +34,13 @@ dependencies=[
 ]
 
 [tool.setuptools.package-data]
-"*" = ["*.fbs", "*.yaml"]
+# @nocommit: prune /test[s]/ dirs, /third-party/ dirs
+# @nocommit: yaml only for kernels?
+"*" = ["*.yaml"]
 
 [tool.setuptools.exclude-package-data]
 "*" = ["*.pyc"]
 
 [tool.usort]
+# Do not try to put "first-party" imports in their own section.
 first_party_detection = false

sdk/CMakeLists.txt

@@ -50,6 +50,9 @@ foreach(schema_file ${_bundled_input_schema_names})
        "${CMAKE_CURRENT_SOURCE_DIR}/bundled_program/schema/${schema_file}")
 endforeach()
 
+set(FLATCC_TEST OFF CACHE BOOL "")
+set(FLATCC_CXX_TEST OFF CACHE BOOL "")
+set(FLATCC_REFLECTION OFF CACHE BOOL "")
 add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../third-party/flatcc
                  ${CMAKE_BINARY_DIR}/third-party/flatcc)
 
@@ -154,6 +157,6 @@ target_include_directories(
 # Install libraries
 install(
   TARGETS bundled_program etdump flatccrt
-  DESTINATION ${CMAKE_BINARY_DIR}/lib
+  DESTINATION ${CMAKE_INSTALL_LIBDIR}
   INCLUDES
   DESTINATION ${_common_include_directories})