put bundled program under executorch/sdk (#1014)

Summary:
Pull Request resolved: #1014

This stack aims to restructure bundledprogram codebase. The target structure should be like D49605947.

This diff moves bundled program codebase from executorch/ to executorch/sdk/

Differential Revision: D50422728

fbshipit-source-id: b22659914ef0278aa8521f3be3b084e6d9edeb45

Author: Gasoonjia

backends/apple/mps/test/test_mps.py

@@ -25,9 +25,9 @@
     TestMPS,
 )
 
-from executorch.bundled_program.config import MethodTestCase, MethodTestSuite
-from executorch.bundled_program.core import create_bundled_program
-from executorch.bundled_program.serialize import (
+from executorch.sdk.bundled_program.config import MethodTestCase, MethodTestSuite
+from executorch.sdk.bundled_program.core import create_bundled_program
+from executorch.sdk.bundled_program.serialize import (
     serialize_from_bundled_program_to_flatbuffer,
 )
 from executorch.exir import ExirExportedProgram

backends/apple/mps/test/test_mps_utils.py

@@ -12,15 +12,15 @@
 
 import torch
 from executorch.backends.apple.mps.mps_preprocess import MPSBackend
-from executorch.bundled_program.config import MethodTestCase, MethodTestSuite
-from executorch.bundled_program.core import create_bundled_program
-from executorch.bundled_program.serialize import (
-    serialize_from_bundled_program_to_flatbuffer,
-)
 from executorch.exir import ExecutorchProgram, ExirExportedProgram
 from executorch.exir.backend.backend_api import to_backend, validation_disabled
 
 from executorch.exir.print_program import print_program
+from executorch.sdk.bundled_program.config import MethodTestCase, MethodTestSuite
+from executorch.sdk.bundled_program.core import create_bundled_program
+from executorch.sdk.bundled_program.serialize import (
+    serialize_from_bundled_program_to_flatbuffer,
+)
 
 # Config for Capturing the weights, will be moved in the future
 _CAPTURE_CONFIG = exir.CaptureConfig(enable_aot=True, _unlift=True)
@@ -191,9 +191,14 @@ def forward(self, *args):
         )
 
         method_test_suites = [
-            MethodTestSuite(method_name="forward", test_cases=[
-                MethodTestCase(input=sample_inputs, expected_outputs=[module(*sample_inputs)])
-            ])
+            MethodTestSuite(
+                method_name="forward",
+                test_cases=[
+                    MethodTestCase(
+                        input=sample_inputs, expected_outputs=[module(*sample_inputs)]
+                    )
+                ],
+            )
         ]
 
         logging.info("  -> Test suites generated successfully")

backends/xnnpack/test/TARGETS

@@ -21,14 +21,14 @@ python_unittest(
         "//executorch/backends/xnnpack:xnnpack_preprocess",
         "//executorch/backends/xnnpack/partition:xnnpack_partitioner",
         "//executorch/backends/xnnpack/utils:xnnpack_utils",
-        "//executorch/bundled_program:config",
-        "//executorch/bundled_program:core",
-        "//executorch/bundled_program/serialize:lib",
         "//executorch/exir:lib",
         "//executorch/exir/backend:backend_api",
         "//executorch/exir/passes:spec_prop_pass",
         "//executorch/extension/pybindings:portable_lib",  # @manual
         "//executorch/extension/pytree:pylib",
+        "//executorch/sdk/bundled_program:config",
+        "//executorch/sdk/bundled_program:core",
+        "//executorch/sdk/bundled_program/serialize:lib",
     ],
 )
 
@@ -49,15 +49,15 @@ python_unittest(
         "//executorch/backends/xnnpack:xnnpack_preprocess",
         "//executorch/backends/xnnpack/partition:xnnpack_partitioner",
         "//executorch/backends/xnnpack/utils:xnnpack_utils",
-        "//executorch/bundled_program:config",
-        "//executorch/bundled_program:core",
-        "//executorch/bundled_program/serialize:lib",
         "//executorch/exir:lib",
         "//executorch/exir/backend:backend_api",
         "//executorch/exir/dialects:lib",
         "//executorch/exir/passes:spec_prop_pass",
         "//executorch/extension/pybindings:portable_lib",  # @manual
         "//executorch/extension/pytree:pylib",
+        "//executorch/sdk/bundled_program:config",
+        "//executorch/sdk/bundled_program:core",
+        "//executorch/sdk/bundled_program/serialize:lib",
     ],
 )
 
@@ -78,14 +78,14 @@ python_unittest(
         "//executorch/backends/xnnpack:xnnpack_preprocess",
         "//executorch/backends/xnnpack/partition:xnnpack_partitioner",
         "//executorch/backends/xnnpack/utils:xnnpack_utils",
-        "//executorch/bundled_program:config",
-        "//executorch/bundled_program:core",
-        "//executorch/bundled_program/serialize:lib",
         "//executorch/exir:lib",
         "//executorch/exir/backend:backend_api",
         "//executorch/exir/passes:spec_prop_pass",
         "//executorch/extension/pybindings:portable_lib",  # @manual
         "//executorch/extension/pytree:pylib",
+        "//executorch/sdk/bundled_program:config",
+        "//executorch/sdk/bundled_program:core",
+        "//executorch/sdk/bundled_program/serialize:lib",
         "//pytorch/vision:torchvision",
     ],
 )

backends/xnnpack/test/test_xnnpack_utils.py

@@ -25,12 +25,6 @@
 
 # import the xnnpack backend implementation
 from executorch.backends.xnnpack.xnnpack_preprocess import XnnpackBackend
-
-from executorch.bundled_program.config import MethodTestCase, MethodTestSuite
-from executorch.bundled_program.core import create_bundled_program
-from executorch.bundled_program.serialize import (
-    serialize_from_bundled_program_to_flatbuffer,
-)
 from executorch.exir import ExecutorchProgram, ExirExportedProgram
 from executorch.exir.backend.backend_api import to_backend, validation_disabled
 
@@ -41,6 +35,12 @@
 )
 from executorch.extension.pytree import tree_flatten
 
+from executorch.sdk.bundled_program.config import MethodTestCase, MethodTestSuite
+from executorch.sdk.bundled_program.core import create_bundled_program
+from executorch.sdk.bundled_program.serialize import (
+    serialize_from_bundled_program_to_flatbuffer,
+)
+
 from torch.ao.quantization import (  # @manual
     default_per_channel_symmetric_qnnpack_qconfig,
     PlaceholderObserver,

docs/source/sdk-bundled-io.md

@@ -86,9 +86,9 @@ Here is a flow highlighting how to generate a `BundledProgram` given a PyTorch m
 
 import torch
 
-from executorch.bundled_program.config import MethodTestCase, MethodTestSuite
-from executorch.bundled_program.core import create_bundled_program
-from executorch.bundled_program.serialize import (
+from executorch.sdk.bundled_program.config import MethodTestCase, MethodTestSuite
+from executorch.sdk.bundled_program.core import create_bundled_program
+from executorch.sdk.bundled_program.serialize import (
     serialize_from_bundled_program_to_flatbuffer,
 )
 
@@ -195,7 +195,7 @@ with open(save_path, "wb") as f:
 We can also regenerate `BundledProgram` from flatbuffer file if needed:
 
 ```python
-from executorch.bundled_program.serialize import deserialize_from_flatbuffer_to_bundled_program
+from executorch.sdk.bundled_program.serialize import deserialize_from_flatbuffer_to_bundled_program
 save_path = "bundled_program.bp"
 with open(save_path, "rb") as f:
     serialized_bundled_program = f.read()
@@ -325,8 +325,8 @@ Here's the example of the dtype of test input not meet model's requirement:
 import torch
 from torch.export import export
 
-from executorch.bundled_program.config import BundledConfig
-from executorch.bundled_program.core import create_bundled_program
+from executorch.sdk.bundled_program.config import BundledConfig
+from executorch.sdk.bundled_program.core import create_bundled_program
 
 from executorch.exir import to_edge
 
@@ -449,8 +449,8 @@ Another common error would be the method name in `BundledConfig` does not exist
 import torch
 from torch.export import export
 
-from executorch.bundled_program.config import BundledConfig
-from executorch.bundled_program.core import create_bundled_program
+from executorch.sdk.bundled_program.config import BundledConfig
+from executorch.sdk.bundled_program.core import create_bundled_program
 
 from executorch.exir import to_edge
 

examples/apple/mps/scripts/mps_example.py

@@ -13,9 +13,9 @@
 import torch._export as export
 from executorch import exir
 from executorch.backends.apple.mps.mps_preprocess import MPSBackend
-from executorch.bundled_program.config import BundledConfig
-from executorch.bundled_program.core import create_bundled_program
-from executorch.bundled_program.serialize import (
+from executorch.sdk.bundled_program.config import BundledConfig
+from executorch.sdk.bundled_program.core import create_bundled_program
+from executorch.sdk.bundled_program.serialize import (
     serialize_from_bundled_program_to_flatbuffer,
 )
 

examples/sdk/scripts/export_bundled_program.py

@@ -11,16 +11,16 @@
 from typing import List
 
 import torch
-from executorch.bundled_program.config import (
+from executorch.exir.schema import Program
+from executorch.sdk.bundled_program.config import (
     MethodInputType,
     MethodTestCase,
     MethodTestSuite,
 )
-from executorch.bundled_program.core import create_bundled_program
-from executorch.bundled_program.serialize import (
+from executorch.sdk.bundled_program.core import create_bundled_program
+from executorch.sdk.bundled_program.serialize import (
     serialize_from_bundled_program_to_flatbuffer,
 )
-from executorch.exir.schema import Program
 
 from ...models import MODEL_NAME_TO_MODEL
 from ...models.model_factory import EagerModelFactory

exir/_serialize/TARGETS

@@ -14,7 +14,7 @@ cpp_python_extension(
         "//executorch/backends/fb/qnnpack/...",
         "//executorch/backends/vulkan/...",
         "//executorch/backends/xnnpack/...",
-        "//executorch/bundled_program/...",
+        "//executorch/sdk/bundled_program/...",
         "//executorch/sdk/etdump/...",
     ],
     deps = [
@@ -43,9 +43,6 @@ runtime.python_library(
     # Please ask before changing this.
     visibility = [
         "//executorch/backends/...",
-        "//executorch/bundled_program:core",
-        "//executorch/bundled_program/serialize:lib",
-        "//executorch/bundled_program/tests/...",
         "//executorch/codegen/...",
         "//executorch/examples/async_exec:emit_program_lib",
         "//executorch/exir/...",
@@ -54,6 +51,9 @@ runtime.python_library(
         "//executorch/extension/pybindings/test:test",
         "//executorch/extension/pybindings/test:test-library",
         "//executorch/profiler/...",
+        "//executorch/sdk/bundled_program:core",
+        "//executorch/sdk/bundled_program/serialize:lib",
+        "//executorch/sdk/bundled_program/tests/...",
         "//executorch/sdk/experimental/...",
         "//executorch/test/...",
         "@EXECUTORCH_CLIENTS",

schema/targets.bzl

@@ -61,13 +61,13 @@ def define_common_targets():
     runtime.export_file(
         name = INPUT_BUNDLED,
         visibility = [
-            "//executorch/bundled_program/serialize/...",
+            "//executorch/sdk/bundled_program/serialize/...",
         ],
     )
     runtime.export_file(
         name = INPUT_SCALAR_TYPE,
         visibility = [
-            "//executorch/bundled_program/serialize/...",
+            "//executorch/sdk/bundled_program/serialize/...",
             "//executorch/exir/_serialize/...",
             "//executorch/sdk/etdump/...",
         ],
@@ -110,7 +110,7 @@ def define_common_targets():
         name = BUNDLED_LIBRARY_NAME,
         srcs = [],
         visibility = [
-            "//executorch/bundled_program/...",
+            "//executorch/sdk/bundled_program/...",
             "//executorch/extension/pybindings/...",
             "//executorch/examples/bundled_executor_runner/...",
             "//executorch/util/...",  # bundled_program_verification

bundled_program/TARGETS renamed to sdk/bundled_program/TARGETS

@@ -8,17 +8,18 @@
 import typing
 from typing import Dict, List, Sequence, Type
 
-import executorch.bundled_program.schema as bp_schema
 import executorch.exir.schema as core_schema
 
+import executorch.sdk.bundled_program.schema as bp_schema
+
 import torch
 import torch.fx
-from executorch.bundled_program.config import ConfigValue, MethodTestSuite
-
-from executorch.bundled_program.version import BUNDLED_PROGRAM_SCHEMA_VERSION
 from executorch.exir._serialize import _serialize_pte_binary
 
 from executorch.exir.tensor import get_scalar_type, scalar_type_enum, TensorSpec
+from executorch.sdk.bundled_program.config import ConfigValue, MethodTestSuite
+
+from executorch.sdk.bundled_program.version import BUNDLED_PROGRAM_SCHEMA_VERSION
 
 # pyre-ignore
 supported_program_type_table: Dict[Type[core_schema.KernelTypes], ConfigValue] = {

bundled_program/config.py renamed to sdk/bundled_program/config.py

@@ -19,19 +19,19 @@ runtime.python_library(
     visibility = [
         "//executorch/bacends/...",
         "//executorch/backends/xnnpack/test/...",
-        "//executorch/bundled_program/tests/...",
         "//executorch/codegen/...",
         "//executorch/examples/async_exec:emit_program_lib",
         "//executorch/exir:lib",
         "//executorch/extension/pybindings/test:test",
         "//executorch/extension/pybindings/test:test-library",
         "//executorch/profiler/...",
+        "//executorch/sdk/bundled_program/tests/...",
         "//executorch/test/...",
         "@EXECUTORCH_CLIENTS",
     ],
     deps = [
         "fbsource//third-party/pypi/setuptools:setuptools",
-        "//executorch/bundled_program:schema",
         "//executorch/exir/_serialize:lib",
+        "//executorch/sdk/bundled_program:schema",
     ],
 )

bundled_program/core.py renamed to sdk/bundled_program/core.py

@@ -14,10 +14,10 @@
 
 # @manual=fbsource//third-party/pypi/setuptools:setuptools
 import pkg_resources
-from executorch.bundled_program.schema import BundledProgram
 
 from executorch.exir._serialize._dataclass import _DataclassEncoder, _json_to_dataclass
 from executorch.exir._serialize._flatbuffer import _flatc_compile, _flatc_decompile
+from executorch.sdk.bundled_program.schema import BundledProgram
 
 # The prefix of schema files used for bundled program
 BUNDLED_PROGRAM_SCHEMA_NAME = "bundled_program_schema"

bundled_program/schema.py renamed to sdk/bundled_program/schema.py

@@ -14,9 +14,9 @@ python_library(
     deps = [
         "fbsource//third-party/pypi/typing-extensions:typing-extensions",
         "//caffe2:torch",
-        "//executorch/bundled_program:config",
         "//executorch/exir:lib",
         "//executorch/exir:schema",
+        "//executorch/sdk/bundled_program:config",
     ],
 )
 
@@ -28,10 +28,10 @@ python_unittest(
     deps = [
         ":lib",
         "//caffe2:torch",
-        "//executorch/bundled_program:config",
-        "//executorch/bundled_program:core",
-        "//executorch/bundled_program:schema",
         "//executorch/exir/_serialize:lib",
+        "//executorch/sdk/bundled_program:config",
+        "//executorch/sdk/bundled_program:core",
+        "//executorch/sdk/bundled_program:schema",
     ],
 )
 
@@ -42,9 +42,9 @@ python_unittest(
     ],
     deps = [
         ":lib",
-        "//executorch/bundled_program:core",
-        "//executorch/bundled_program/serialize:lib",
         "//executorch/exir:print_program",
+        "//executorch/sdk/bundled_program:core",
+        "//executorch/sdk/bundled_program/serialize:lib",
     ],
 )
 
@@ -56,8 +56,8 @@ python_unittest(
     deps = [
         ":lib",
         "//caffe2:torch",
-        "//executorch/bundled_program:config",
         "//executorch/extension/pytree:pylib",
+        "//executorch/sdk/bundled_program:config",
     ],
 )
 
@@ -69,9 +69,6 @@ python_unittest(
     deps = [
         ":lib",
         "//caffe2:torch",
-        "//executorch/bundled_program:config",
-        "//executorch/bundled_program:core",
-        "//executorch/bundled_program/serialize:lib",
         "//executorch/exir:dynamic_shape",
         "//executorch/exir:lib",
         "//executorch/exir:memory",
@@ -88,5 +85,8 @@ python_unittest(
         "//executorch/extension/pybindings:portable_lib",
         "//executorch/extension/pytree:pybindings",
         "//executorch/kernels/portable:custom_ops_generated_lib",
+        "//executorch/sdk/bundled_program:config",
+        "//executorch/sdk/bundled_program:core",
+        "//executorch/sdk/bundled_program/serialize:lib",
     ],
 )