move apis for BP creation under bundled_program/bundler (#1019)

Summary:
Pull Request resolved: #1019

Pull Request resolved: #435

This stack focuses on moving all bundled program codes under executorch/sdk/bundled_program. Target and discussions can be found in D49605947.

This diff moves all apis for bundled program creation under bundled_program/bundler, and update dependencies.

Differential Revision: https://internalfb.com/D50422556

fbshipit-source-id: 8b2b56a81d7bef6c34e4e0e539d113ddb7980b6d

Author: Songhao Jia

backends/apple/mps/test/test_mps.py

@@ -25,8 +25,8 @@
     TestMPS,
 )
 
-from executorch.sdk.bundled_program.config import MethodTestCase, MethodTestSuite
-from executorch.sdk.bundled_program.core import create_bundled_program
+from executorch.sdk.bundled_program.bundler.config import MethodTestCase, MethodTestSuite
+from executorch.sdk.bundled_program.bundler.core import create_bundled_program
 from executorch.sdk.bundled_program.serialize import (
     serialize_from_bundled_program_to_flatbuffer,
 )

backends/apple/mps/test/test_mps_utils.py

@@ -16,8 +16,8 @@
 from executorch.exir.backend.backend_api import to_backend, validation_disabled
 
 from executorch.exir.print_program import print_program
-from executorch.sdk.bundled_program.config import MethodTestCase, MethodTestSuite
-from executorch.sdk.bundled_program.core import create_bundled_program
+from executorch.sdk.bundled_program.bundler.config import MethodTestCase, MethodTestSuite
+from executorch.sdk.bundled_program.bundler.core import create_bundled_program
 from executorch.sdk.bundled_program.serialize import (
     serialize_from_bundled_program_to_flatbuffer,
 )

backends/xnnpack/test/TARGETS

@@ -26,8 +26,8 @@ python_unittest(
         "//executorch/exir/passes:spec_prop_pass",
         "//executorch/extension/pybindings:portable_lib",  # @manual
         "//executorch/extension/pytree:pylib",
-        "//executorch/sdk/bundled_program:config",
-        "//executorch/sdk/bundled_program:core",
+        "//executorch/sdk/bundled_program/bundler:config",
+        "//executorch/sdk/bundled_program/bundler:core",
         "//executorch/sdk/bundled_program/serialize:lib",
     ],
 )
@@ -55,8 +55,8 @@ python_unittest(
         "//executorch/exir/passes:spec_prop_pass",
         "//executorch/extension/pybindings:portable_lib",  # @manual
         "//executorch/extension/pytree:pylib",
-        "//executorch/sdk/bundled_program:config",
-        "//executorch/sdk/bundled_program:core",
+        "//executorch/sdk/bundled_program/bundler:config",
+        "//executorch/sdk/bundled_program/bundler:core",
         "//executorch/sdk/bundled_program/serialize:lib",
     ],
 )
@@ -83,8 +83,8 @@ python_unittest(
         "//executorch/exir/passes:spec_prop_pass",
         "//executorch/extension/pybindings:portable_lib",  # @manual
         "//executorch/extension/pytree:pylib",
-        "//executorch/sdk/bundled_program:config",
-        "//executorch/sdk/bundled_program:core",
+        "//executorch/sdk/bundled_program/bundler:config",
+        "//executorch/sdk/bundled_program/bundler:core",
         "//executorch/sdk/bundled_program/serialize:lib",
         "//pytorch/vision:torchvision",
     ],

backends/xnnpack/test/test_xnnpack_utils.py

@@ -35,8 +35,11 @@
 )
 from executorch.extension.pytree import tree_flatten
 
-from executorch.sdk.bundled_program.config import MethodTestCase, MethodTestSuite
-from executorch.sdk.bundled_program.core import create_bundled_program
+from executorch.sdk.bundled_program.bundler.config import (
+    MethodTestCase,
+    MethodTestSuite,
+)
+from executorch.sdk.bundled_program.bundler.core import create_bundled_program
 from executorch.sdk.bundled_program.serialize import (
     serialize_from_bundled_program_to_flatbuffer,
 )

docs/source/sdk-bundled-io.md

@@ -86,8 +86,8 @@ Here is a flow highlighting how to generate a `BundledProgram` given a PyTorch m
 
 import torch
 
-from executorch.sdk.bundled_program.config import MethodTestCase, MethodTestSuite
-from executorch.sdk.bundled_program.core import create_bundled_program
+from executorch.sdk.bundled_program.bundler.config import MethodTestCase, MethodTestSuite
+from executorch.sdk.bundled_program.bundler.core import create_bundled_program
 from executorch.sdk.bundled_program.serialize import (
     serialize_from_bundled_program_to_flatbuffer,
 )
@@ -324,8 +324,8 @@ Here's the example of the dtype of test input not meet model's requirement:
 ```python
 import torch
 
-from executorch.sdk.bundled_program.config import MethodTestCase, MethodTestSuite
-from executorch.sdk.bundled_program.core import create_bundled_program
+from executorch.sdk.bundled_program.bundler.config import MethodTestCase, MethodTestSuite
+from executorch.sdk.bundled_program.bundler.core import create_bundled_program
 
 from executorch.exir import to_edge
 from torch.export import export
@@ -459,8 +459,8 @@ Another common error would be the method name in any `MethodTestSuite` does not
 ```python
 import torch
 
-from executorch.sdk.bundled_program.config import MethodTestCase, MethodTestSuite
-from executorch.sdk.bundled_program.core import create_bundled_program
+from executorch.sdk.bundled_program.bundler.config import MethodTestCase, MethodTestSuite
+from executorch.sdk.bundled_program.bundler.core import create_bundled_program
 
 from executorch.exir import to_edge
 from torch.export import export

examples/apple/mps/scripts/mps_example.py

@@ -13,8 +13,8 @@
 import torch._export as export
 from executorch import exir
 from executorch.backends.apple.mps.mps_preprocess import MPSBackend
-from executorch.sdk.bundled_program.config import MethodTestCase, MethodTestSuite
-from executorch.sdk.bundled_program.core import create_bundled_program
+from executorch.sdk.bundled_program.bundler.config import MethodTestCase, MethodTestSuite
+from executorch.sdk.bundled_program.bundler.core import create_bundled_program
 from executorch.sdk.bundled_program.serialize import (
     serialize_from_bundled_program_to_flatbuffer,
 )

examples/sdk/scripts/export_bundled_program.py

@@ -12,12 +12,12 @@
 
 import torch
 from executorch.exir.schema import Program
-from executorch.sdk.bundled_program.config import (
+from executorch.sdk.bundled_program.bundler.config import (
     MethodInputType,
     MethodTestCase,
     MethodTestSuite,
 )
-from executorch.sdk.bundled_program.core import create_bundled_program
+from executorch.sdk.bundled_program.bundler.core import create_bundled_program
 from executorch.sdk.bundled_program.serialize import (
     serialize_from_bundled_program_to_flatbuffer,
 )

exir/_serialize/TARGETS

@@ -51,7 +51,7 @@ runtime.python_library(
         "//executorch/extension/pybindings/test:test",
         "//executorch/extension/pybindings/test:test-library",
         "//executorch/profiler/...",
-        "//executorch/sdk/bundled_program:core",
+        "//executorch/sdk/bundled_program/bundler:core",
         "//executorch/sdk/bundled_program/serialize:lib",
         "//executorch/sdk/bundled_program/tests/...",
         "//executorch/sdk/experimental/...",

sdk/bundled_program/TARGETS

@@ -2,33 +2,6 @@ load("@fbcode_macros//build_defs:python_library.bzl", "python_library")
 
 oncall("executorch")
 
-python_library(
-    name = "core",
-    srcs = [
-        "core.py",
-    ],
-    deps = [
-        ":config",
-        ":schema",
-        ":version",
-        "//caffe2:torch",
-        "//executorch/exir:schema",
-        "//executorch/exir:tensor",
-        "//executorch/exir/_serialize:lib",
-    ],
-)
-
-python_library(
-    name = "config",
-    srcs = [
-        "config.py",
-    ],
-    deps = [
-        "fbsource//third-party/pypi/typing-extensions:typing-extensions",
-        "//caffe2:torch",
-    ],
-)
-
 python_library(
     name = "schema",
     srcs = [
@@ -38,10 +11,3 @@ python_library(
         "//executorch/exir:scalar_type",
     ],
 )
-
-python_library(
-    name = "version",
-    srcs = [
-        "version.py",
-    ],
-)

sdk/bundled_program/bundler/TARGETS

@@ -0,0 +1,37 @@
+load("@fbcode_macros//build_defs:python_library.bzl", "python_library")
+
+oncall("executorch")
+
+python_library(
+    name = "core",
+    srcs = [
+        "core.py",
+    ],
+    deps = [
+        ":config",
+        ":version",
+        "//caffe2:torch",
+        "//executorch/exir:schema",
+        "//executorch/exir:tensor",
+        "//executorch/exir/_serialize:lib",
+        "//executorch/sdk/bundled_program:schema",
+    ],
+)
+
+python_library(
+    name = "config",
+    srcs = [
+        "config.py",
+    ],
+    deps = [
+        "fbsource//third-party/pypi/typing-extensions:typing-extensions",
+        "//caffe2:torch",
+    ],
+)
+
+python_library(
+    name = "version",
+    srcs = [
+        "version.py",
+    ],
+)

sdk/bundled_program/config.py renamed to sdk/bundled_program/bundler/config.py

@@ -17,9 +17,11 @@
 from executorch.exir._serialize import _serialize_pte_binary
 
 from executorch.exir.tensor import get_scalar_type, scalar_type_enum, TensorSpec
-from executorch.sdk.bundled_program.config import ConfigValue, MethodTestSuite
+from executorch.sdk.bundled_program.bundler.config import ConfigValue, MethodTestSuite
 
-from executorch.sdk.bundled_program.version import BUNDLED_PROGRAM_SCHEMA_VERSION
+from executorch.sdk.bundled_program.bundler.version import (
+    BUNDLED_PROGRAM_SCHEMA_VERSION,
+)
 
 # pyre-ignore
 supported_program_type_table: Dict[Type[core_schema.KernelTypes], ConfigValue] = {

sdk/bundled_program/core.py renamed to sdk/bundled_program/bundler/core.py

@@ -16,7 +16,7 @@ python_library(
         "//caffe2:torch",
         "//executorch/exir:lib",
         "//executorch/exir:schema",
-        "//executorch/sdk/bundled_program:config",
+        "//executorch/sdk/bundled_program/bundler:config",
     ],
 )
 
@@ -29,9 +29,9 @@ python_unittest(
         ":lib",
         "//caffe2:torch",
         "//executorch/exir/_serialize:lib",
-        "//executorch/sdk/bundled_program:config",
-        "//executorch/sdk/bundled_program:core",
         "//executorch/sdk/bundled_program:schema",
+        "//executorch/sdk/bundled_program/bundler:config",
+        "//executorch/sdk/bundled_program/bundler:core",
     ],
 )
 
@@ -43,7 +43,7 @@ python_unittest(
     deps = [
         ":lib",
         "//executorch/exir:print_program",
-        "//executorch/sdk/bundled_program:core",
+        "//executorch/sdk/bundled_program/bundler:core",
         "//executorch/sdk/bundled_program/serialize:lib",
     ],
 )
@@ -57,7 +57,7 @@ python_unittest(
         ":lib",
         "//caffe2:torch",
         "//executorch/extension/pytree:pylib",
-        "//executorch/sdk/bundled_program:config",
+        "//executorch/sdk/bundled_program/bundler:config",
     ],
 )
 
@@ -85,8 +85,8 @@ python_unittest(
         "//executorch/extension/pybindings:portable_lib",
         "//executorch/extension/pytree:pybindings",
         "//executorch/kernels/portable:custom_ops_generated_lib",
-        "//executorch/sdk/bundled_program:config",
-        "//executorch/sdk/bundled_program:core",
+        "//executorch/sdk/bundled_program/bundler:config",
+        "//executorch/sdk/bundled_program/bundler:core",
         "//executorch/sdk/bundled_program/serialize:lib",
     ],
 )

sdk/bundled_program/version.py renamed to sdk/bundled_program/bundler/version.py

@@ -10,9 +10,10 @@
 from typing import List, Tuple
 
 import executorch.exir as exir
+
 import torch
 from executorch.exir.schema import Program
-from executorch.sdk.bundled_program.config import (
+from executorch.sdk.bundled_program.bundler.config import (
     MethodInputType,
     MethodOutputType,
     MethodTestCase,

sdk/bundled_program/tests/TARGETS

@@ -13,8 +13,8 @@
 
 import torch
 from executorch.exir._serialize import _serialize_pte_binary
-from executorch.sdk.bundled_program.config import ConfigValue
-from executorch.sdk.bundled_program.core import create_bundled_program
+from executorch.sdk.bundled_program.bundler.config import ConfigValue
+from executorch.sdk.bundled_program.bundler.core import create_bundled_program
 from executorch.sdk.bundled_program.tests.common import get_common_program
 
 

sdk/bundled_program/tests/common.py

@@ -11,7 +11,7 @@
 
 import torch
 from executorch.extension.pytree import tree_flatten
-from executorch.sdk.bundled_program.config import DataContainer
+from executorch.sdk.bundled_program.bundler.config import DataContainer
 
 from executorch.sdk.bundled_program.tests.common import (
     get_random_test_suites,

sdk/bundled_program/tests/test_bundle_data.py

@@ -21,7 +21,7 @@
 
 import torch
 
-from executorch.sdk.bundled_program.core import create_bundled_program
+from executorch.sdk.bundled_program.bundler.core import create_bundled_program
 from executorch.sdk.bundled_program.serialize import (
     serialize_from_bundled_program_to_flatbuffer,
 )

sdk/bundled_program/tests/test_config.py

@@ -10,7 +10,7 @@
 
 from executorch.exir.print_program import pretty_print
 
-from executorch.sdk.bundled_program.core import create_bundled_program
+from executorch.sdk.bundled_program.bundler.core import create_bundled_program
 
 from executorch.sdk.bundled_program.serialize import (
     deserialize_from_flatbuffer_to_bundled_program,

sdk/bundled_program/tests/test_end2end.py

@@ -56,8 +56,8 @@ python_unittest(
         "//executorch/extension/pybindings:aten_lib",
         "//executorch/extension/pytree:pybindings",
         "//executorch/kernels/portable:custom_ops_generated_lib",
-        "//executorch/sdk/bundled_program:config",
-        "//executorch/sdk/bundled_program:core",
+        "//executorch/sdk/bundled_program/bundler:config",
+        "//executorch/sdk/bundled_program/bundler:core",
         "//executorch/sdk/bundled_program/serialize:lib",
     ],
 )
@@ -88,8 +88,8 @@ python_unittest(
         "//executorch/extension/pybindings:portable_lib",
         "//executorch/extension/pytree:pybindings",
         "//executorch/kernels/portable:custom_ops_generated_lib",
-        "//executorch/sdk/bundled_program:config",
-        "//executorch/sdk/bundled_program:core",
+        "//executorch/sdk/bundled_program/bundler:config",
+        "//executorch/sdk/bundled_program/bundler:core",
         "//executorch/sdk/bundled_program/serialize:lib",
     ],
 )

sdk/bundled_program/tests/test_serialize.py

@@ -22,8 +22,11 @@
 from executorch.exir import ExecutorchBackendConfig
 from executorch.exir.passes import MemoryPlanningPass, ToOutVarPass
 from executorch.exir.print_program import pretty_print
-from executorch.sdk.bundled_program.config import MethodTestCase, MethodTestSuite
-from executorch.sdk.bundled_program.core import create_bundled_program
+from executorch.sdk.bundled_program.bundler.config import (
+    MethodTestCase,
+    MethodTestSuite,
+)
+from executorch.sdk.bundled_program.bundler.core import create_bundled_program
 from executorch.sdk.bundled_program.serialize import (
     serialize_from_bundled_program_to_flatbuffer,
 )

test/end2end/TARGETS

@@ -22,8 +22,8 @@ def define_common_targets():
         deps = [
             ":linear_model",
             "//caffe2:torch",
-            "//executorch/sdk/bundled_program:config",
-            "//executorch/sdk/bundled_program:core",
+            "//executorch/sdk/bundled_program/bundler:config",
+            "//executorch/sdk/bundled_program/bundler:core",
             "//executorch/sdk/bundled_program/serialize:lib",
             "//executorch/exir:lib",
             "//executorch/exir/_serialize:lib",