fb/ pybindings

Summary: Before we had 2 pybinding libs. Portable and ATen. Now we have 4. Portable, Portable+custom, ATen, ATen + Custom. The + Custom variants are internal only (as they include all the internal custom ops).

Reviewed By: dbort

Differential Revision: D47775647

fbshipit-source-id: a7cda8cc051304ed980c19560ae32d2feb3bb505

Author: JacobSzwejbka

backends/test/demos/rpc/targets.bzl

@@ -4,7 +4,7 @@ load(
     "CXX",
 )
 load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
-load("@fbsource//xplat/executorch/extension/pybindings:targets.bzl", "MODELS_ALL_OPS_LEAN_MODE_GENERATED_LIB")
+load("@fbsource//xplat/executorch/extension/pybindings:targets.bzl", "MODELS_ATEN_OPS_LEAN_MODE_GENERATED_LIB")
 
 def define_common_targets():
     """Defines targets that should be shared between fbcode and xplat.
@@ -28,7 +28,7 @@ def define_common_targets():
             "//executorch/runtime/backend:backend_registry",
             "//executorch/extension/data_loader:buffer_data_loader",
             "//executorch/util:util",
-        ] + MODELS_ALL_OPS_LEAN_MODE_GENERATED_LIB,
+        ] + MODELS_ATEN_OPS_LEAN_MODE_GENERATED_LIB,
         exported_deps = [
             "//executorch/runtime/core:core",
         ],

extension/pybindings/TARGETS

@@ -2,7 +2,7 @@
 # Any targets that should be shared between fbcode and xplat must be defined in
 # targets.bzl. This file can contain fbcode-only targets.
 
-load("@fbcode//executorch/extension/pybindings:targets.bzl", "MODELS_ALL_OPS_ATEN_MODE_GENERATED_LIB", "MODELS_ALL_OPS_LEAN_MODE_GENERATED_LIB", "MODULE_DEPS", "define_common_targets", "executorch_pybindings")
+load("@fbcode//executorch/extension/pybindings:targets.bzl", "MODELS_ATEN_OPS_ATEN_MODE_GENERATED_LIB", "MODELS_ATEN_OPS_LEAN_MODE_GENERATED_LIB", "MODULE_DEPS", "define_common_targets", "executorch_pybindings")
 load("@fbcode_macros//build_defs:python_library.bzl", "python_library")
 
 define_common_targets()
@@ -20,7 +20,7 @@ executorch_pybindings(
     srcs = [
         "module.cpp",
     ],
-    cppdeps = MODULE_DEPS + MODELS_ALL_OPS_LEAN_MODE_GENERATED_LIB,
+    cppdeps = MODULE_DEPS + MODELS_ATEN_OPS_LEAN_MODE_GENERATED_LIB,
     python_module_name = "portable",
     visibility = ["PUBLIC"],
 )
@@ -42,7 +42,7 @@ executorch_pybindings(
         "//executorch/util:bundled_program_verification_aten",
         "//caffe2:torch-cpp",
         "//executorch/runtime/executor/test:test_backend_compiler_lib_aten",
-    ] + MODELS_ALL_OPS_ATEN_MODE_GENERATED_LIB,
+    ] + MODELS_ATEN_OPS_ATEN_MODE_GENERATED_LIB,
     python_module_name = "aten_mode_lib",
     visibility = ["PUBLIC"],
 )

extension/pybindings/targets.bzl

@@ -6,14 +6,6 @@ MODELS_ATEN_OPS_LEAN_MODE_GENERATED_LIB = [
     "//executorch/kernels/portable:generated_lib",
 ]
 
-# Custom ops with portable kernel
-MODELS_CUSTOM_OPS_LEAN_MODE_GENERATED_LIB = [
-    "//executorch/kernels/quantized:generated_lib",
-    "//pye/model_inventory/asr_models/runtime:generated_custom_op_lib_lean",
-]
-
-MODELS_ALL_OPS_LEAN_MODE_GENERATED_LIB = MODELS_ATEN_OPS_LEAN_MODE_GENERATED_LIB + MODELS_CUSTOM_OPS_LEAN_MODE_GENERATED_LIB
-
 MODULE_DEPS = [
     "//caffe2:ATen",
     "//caffe2:torch",
@@ -37,22 +29,11 @@ MODELS_ATEN_OPS_ATEN_MODE_GENERATED_LIB = [
     "//executorch/kernels/aten:generated_lib_aten",
 ]
 
-# Generated libs for all ATen ops AND custom ops used by models in //pye/model_inventory
-MODELS_ALL_OPS_ATEN_MODE_GENERATED_LIB = MODELS_ATEN_OPS_ATEN_MODE_GENERATED_LIB + [
-    "//caffe2/fb/custom_ops/turing:turing_lib_aten",
-    "//pye/model_inventory/asr_models/runtime:generated_lib_aten",
-    "//pye/model_inventory/asr_models/runtime:custom_ops_generated_lib_aten",
-    "//pye/model_inventory/fam_models/runtime:generated_lib_aten",
-    "//pye/model_inventory/ocr_detection_model_non_quantized/runtime:generated_lib_aten",
-    "//caffe2/fb/custom_ops/nimble/et_runtime:generated_lib_aten",
-    "//pye/model_inventory/keyboard_tracking_model/runtime:generated_lib_aten",
-]
-
 def executorch_pybindings(python_module_name, srcs = [], cppdeps = [], visibility = ["//executorch/..."]):
     runtime.cxx_python_extension(
         name = python_module_name,
         srcs = [
-            "pybindings.cpp",
+            "//executorch/extension/pybindings:pybindings.cpp",
         ] + srcs,
         base_module = "executorch.extension.pybindings",
         preprocessor_flags = [
@@ -78,6 +59,18 @@ def define_common_targets():
     TARGETS and BUCK files that call this function.
     """
 
+    # Export these so the internal fb/ subdir can create pybindings with custom internal deps
+    # without forking the pybinding source.
+    runtime.export_file(
+        name = "pybindings.cpp",
+        visibility = ["//executorch/extension/pybindings/..."],
+    )
+
+    runtime.export_file(
+        name = "module.cpp",
+        visibility = ["//executorch/extension/pybindings/..."],
+    )
+
     executorch_pybindings(
         srcs = [
             "module_stub.cpp",

sdk/runners/targets.bzl

@@ -4,7 +4,7 @@ load(
     "CXX",
 )
 load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
-load("@fbsource//xplat/executorch/extension/pybindings:targets.bzl", "MODELS_ALL_OPS_ATEN_MODE_GENERATED_LIB", "MODELS_ALL_OPS_LEAN_MODE_GENERATED_LIB")
+load("@fbsource//xplat/executorch/extension/pybindings:targets.bzl", "MODELS_ATEN_OPS_ATEN_MODE_GENERATED_LIB", "MODELS_ATEN_OPS_LEAN_MODE_GENERATED_LIB")
 
 def define_common_targets():
     """Defines targets that should be shared between fbcode and xplat.
@@ -27,9 +27,9 @@ def define_common_targets():
                 "//executorch/extension/data_loader:buffer_data_loader",
                 "//executorch/extension/data_loader:file_data_loader",
                 "//executorch/util:util" + aten_suffix,
-            ] + (MODELS_ALL_OPS_ATEN_MODE_GENERATED_LIB if aten_mode else [
+            ] + (MODELS_ATEN_OPS_ATEN_MODE_GENERATED_LIB if aten_mode else [
                 "//executorch/configurations:executor_cpu_optimized",
-            ] + MODELS_ALL_OPS_LEAN_MODE_GENERATED_LIB),
+            ] + MODELS_ATEN_OPS_LEAN_MODE_GENERATED_LIB),
             preprocessor_flags = ["-DUSE_ATEN_LIB"] if aten_mode else [],
             external_deps = [
                 "gflags",

shim/xplat/executorch/build/runtime_wrapper.bzl

@@ -267,6 +267,7 @@ def _cxx_test(*args, **kwargs):
 
 def _cxx_python_extension(*args, **kwargs):
     _patch_kwargs_common(kwargs)
+    kwargs["srcs"] = _patch_executorch_references(kwargs["srcs"])
     env.cxx_python_extension(*args, **kwargs)
 
 def _export_file(*args, **kwargs):