Skip to content

Commit 03cd46e

Browse files
Gasoonjiafacebook-github-bot
Gasoonjia
authored and
facebook-github-bot
committed
move schema files under bundled_program/schema (#1057)
Summary: Pull Request resolved: #1057 Pull Request resolved: #436 This stack focuses on moving all bundled program codes under executorch/sdk/bundled_program. Target and discussions can be found in D49605947. This diff moves all bundled program schemas, both py schema and fbs schema, under bundled_program/schema, and update dependencies. Differential Revision: D50422560 fbshipit-source-id: 37490091f2c573af86ff4733f9e6136aff5073a0
1 parent 7aeac2f commit 03cd46e

File tree

17 files changed

+204
-62
lines changed

17 files changed

+204
-62
lines changed

‎extension/pybindings/pybindings.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,8 @@
2525
#include <executorch/runtime/platform/assert.h>
2626
#include <executorch/runtime/platform/profiler.h>
2727
#include <executorch/runtime/platform/runtime.h>
28-
#include <executorch/schema/bundled_program_schema_generated.h>
2928
#include <executorch/sdk/bundled_program/runtime/bundled_program_verification.h>
29+
#include <executorch/sdk/bundled_program/schema/bundled_program_schema_generated.h>
3030
#include <executorch/util/read_file.h>
3131

3232
#include <ATen/Functions.h>

‎schema/targets.bzl

Lines changed: 1 addition & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -2,22 +2,17 @@ load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
22

33
# Construct the input and output file names. All input and output files rely on scalar_type file.
44
PROGRAM_STEM = "program"
5-
BUNLDED_STEM = "bundled_program_schema"
65
SCALAR_TYPE_STEM = "scalar_type"
76

87
INPUT_PROGRAM = PROGRAM_STEM + ".fbs"
9-
INPUT_BUNDLED = BUNLDED_STEM + ".fbs"
108
INPUT_SCALAR_TYPE = SCALAR_TYPE_STEM + ".fbs"
119

1210
OUTPUT_PROGRAM_HEADER = PROGRAM_STEM + "_generated.h"
13-
OUTPUT_BUNDLED_HEADER = BUNLDED_STEM + "_generated.h"
1411
OUTPUT_SCALAR_TYPE_HEADER = SCALAR_TYPE_STEM + "_generated.h"
1512

1613
PROGRAM_GEN_RULE_NAME = "generate_program"
17-
BUNDLED_GEN_RULE_NAME = "generate_bundled_program"
1814

1915
PROGRAM_LIRRARY_NAME = PROGRAM_STEM
20-
BUNDLED_LIBRARY_NAME = BUNLDED_STEM
2116

2217
def _generate_schema_header(rule_name, srcs, headers, default_header):
2318
"""Generate header file given flatbuffer schema
@@ -58,17 +53,11 @@ def define_common_targets():
5853
"//executorch/exir/_serialize/...",
5954
],
6055
)
61-
runtime.export_file(
62-
name = INPUT_BUNDLED,
63-
visibility = [
64-
"//executorch/sdk/bundled_program/serialize/...",
65-
],
66-
)
6756
runtime.export_file(
6857
name = INPUT_SCALAR_TYPE,
6958
visibility = [
70-
"//executorch/sdk/bundled_program/serialize/...",
7159
"//executorch/exir/_serialize/...",
60+
"//executorch/sdk/bundled_program/serialize/...",
7261
"//executorch/sdk/etdump/...",
7362
],
7463
)
@@ -80,13 +69,6 @@ def define_common_targets():
8069
OUTPUT_PROGRAM_HEADER,
8170
)
8271

83-
_generate_schema_header(
84-
BUNDLED_GEN_RULE_NAME,
85-
[INPUT_BUNDLED, INPUT_SCALAR_TYPE],
86-
[OUTPUT_BUNDLED_HEADER, OUTPUT_SCALAR_TYPE_HEADER],
87-
OUTPUT_BUNDLED_HEADER,
88-
)
89-
9072
# Header-only library target with the generate executorch program schema header.
9173
runtime.cxx_library(
9274
name = PROGRAM_LIRRARY_NAME,
@@ -95,7 +77,6 @@ def define_common_targets():
9577
# Lock this down as tightly as possible to ensure that flatbuffers
9678
# are an implementation detail. Ideally this list would only include
9779
# //executorch/runtime/executor/...
98-
"//executorch/bundled_program/runtime/...",
9980
"//executorch/codegen/tools/...",
10081
"//executorch/runtime/executor/...",
10182
],
@@ -106,22 +87,6 @@ def define_common_targets():
10687
exported_external_deps = ["flatbuffers-api"],
10788
)
10889

109-
# Header-only library target with the generate bundled program schema header.
110-
runtime.cxx_library(
111-
name = BUNDLED_LIBRARY_NAME,
112-
srcs = [],
113-
visibility = [
114-
"//executorch/sdk/bundled_program/...",
115-
"//executorch/extension/pybindings/...",
116-
"//executorch/examples/bundled_executor_runner/...",
117-
"//executorch/util/...", # bundled_program_verification
118-
],
119-
exported_headers = {
120-
OUTPUT_BUNDLED_HEADER: ":{}[{}]".format(BUNDLED_GEN_RULE_NAME, OUTPUT_BUNDLED_HEADER),
121-
OUTPUT_SCALAR_TYPE_HEADER: ":{}[{}]".format(PROGRAM_GEN_RULE_NAME, OUTPUT_SCALAR_TYPE_HEADER),
122-
},
123-
exported_external_deps = ["flatbuffers-api"],
124-
)
12590

12691
runtime.cxx_library(
12792
name = "extended_header",

‎sdk/bundled_program/TARGETS

Lines changed: 0 additions & 13 deletions
This file was deleted.

‎sdk/bundled_program/bundler/TARGETS

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ python_library(
1414
"//executorch/exir:schema",
1515
"//executorch/exir:tensor",
1616
"//executorch/exir/_serialize:lib",
17-
"//executorch/sdk/bundled_program:schema",
17+
"//executorch/sdk/bundled_program/schema:bundled_program_schema_py",
1818
],
1919
)
2020

‎sdk/bundled_program/bundler/core.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010

1111
import executorch.exir.schema as core_schema
1212

13-
import executorch.sdk.bundled_program.schema as bp_schema
13+
import executorch.sdk.bundled_program.schema.bundled_program_schema as bp_schema
1414

1515
import torch
1616
import torch.fx

‎sdk/bundled_program/runtime/TARGETS

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# Any targets that should be shared between fbcode and xplat must be defined in
2-
# targets.bzl. This file can contain xplat-only targets.
2+
# targets.bzl. This file can contain fbcode-only targets.
33

44
load(":targets.bzl", "define_common_targets")
55

‎sdk/bundled_program/runtime/bundled_program_verification.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
#include <executorch/runtime/core/memory_allocator.h>
2121
#include <executorch/runtime/executor/method.h>
2222
#include <executorch/runtime/platform/log.h>
23-
#include <executorch/schema/bundled_program_schema_generated.h>
23+
#include <executorch/sdk/bundled_program/schema/bundled_program_schema_generated.h>
2424

2525
namespace torch {
2626
namespace executor {

‎sdk/bundled_program/runtime/targets.bzl

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ def define_common_targets():
1919
],
2020
deps = [
2121
"//executorch/runtime/core/exec_aten/util:dim_order_util" + aten_suffix,
22-
"//executorch/schema:bundled_program_schema",
22+
"//executorch/sdk/bundled_program/schema:bundled_program_schema_fbs",
2323
],
2424
exported_deps = [
2525
"//executorch/runtime/core:memory_allocator",

‎sdk/bundled_program/schema/TARGETS

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# Any targets that should be shared between fbcode and xplat must be defined in
2+
# targets.bzl. This file can contain fbcode-only targets.
3+
4+
load("@fbcode_macros//build_defs:python_library.bzl", "python_library")
5+
load(":targets.bzl", "define_common_targets")
6+
7+
define_common_targets()
8+
9+
python_library(
10+
name = "bundled_program_schema_py",
11+
srcs = [
12+
"bundled_program_schema.py",
13+
],
14+
deps = [
15+
"//executorch/exir:scalar_type",
16+
],
17+
)

‎sdk/bundled_program/schema/bundled_program_schema.fbs

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
// Copyright (c) Meta Platforms, Inc. and affiliates.
2+
3+
//
4+
// See README.md before modifying this file.
5+
//
6+
7+
include "scalar_type.fbs";
8+
9+
namespace bundled_program_flatbuffer;
10+
11+
// Identifier of a valid bundled program schema.
12+
file_identifier "BP06";
13+
// Extension of written files.
14+
file_extension "bp";
15+
16+
// Reason for basic struct: union value type can only be table/struct/string
17+
table Int {
18+
int_val:long;
19+
}
20+
21+
table Bool {
22+
bool_val:bool;
23+
}
24+
25+
table Double {
26+
double_val:double;
27+
}
28+
29+
// All information we need to bundle for a tensor EValue input.
30+
table Tensor {
31+
// The scalar type of Tensor
32+
scalar_type: executorch_flatbuffer.ScalarType;
33+
// The target sizes of the tensor.
34+
sizes: [int];
35+
// The contents of the corresponding input tensor.
36+
data: [ubyte] (force_align: 16);
37+
dim_order:[ubyte];
38+
}
39+
40+
union ValueUnion {
41+
Tensor,
42+
Int,
43+
Bool,
44+
Double,
45+
}
46+
47+
// Abstraction for BundledMethodTestCase values
48+
table Value {
49+
val: ValueUnion;
50+
}
51+
52+
// A single test for a method. The provided inputs should produce the
53+
// expected outputs.
54+
table BundledMethodTestCase {
55+
// The inputs to provide to the method. The number and types of inputs must
56+
// match the schema of the method under test.
57+
inputs: [Value];
58+
59+
// The expected outputs generated while running the model in eager mode using
60+
// the inputs provided. Its length should be equal to the length of program
61+
// outputs.
62+
expected_outputs: [Value];
63+
}
64+
65+
// Collection of test cases for a program method.
66+
table BundledMethodTestSuite {
67+
// The name of the method to test; e.g., "forward" for the forward() method
68+
// of an nn.Module. This name match a method defined by the ExecuTorch
69+
// program.
70+
method_name: string;
71+
72+
// Individual test cases for the method.
73+
test_cases: [BundledMethodTestCase];
74+
}
75+
76+
77+
// Executorch program bunlded with data for verification.
78+
table BundledProgram {
79+
// Schema version.
80+
version:uint;
81+
82+
// Test sets to run against the program.
83+
// Each BundledMethodTestSuite should be used for the method of program sharing same name.
84+
method_test_suites: [BundledMethodTestSuite];
85+
86+
// The binary data of a serialized Executorch program.
87+
// The following `force_align` may sliently override any larger force_align
88+
// used in the program. Therefore, to keep the data (including constant
89+
// tensor, delegate data, etc, see schema.fbs for more info) in the
90+
// executorch program keeps the same alignment as original no matter how
91+
// the program schema changes, we need to make the force_align here the max
92+
// one around all kinds of force_align in the current and future program
93+
// schema, so we use the 4096 as the force_align here.
94+
program: [ubyte] (force_align: 4096);
95+
}
96+
97+
root_type BundledProgram;

‎sdk/bundled_program/schema.py renamed to ‎sdk/bundled_program/schema/bundled_program_schema.py

File renamed without changes.

‎sdk/bundled_program/schema/targets.bzl

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
2+
3+
BUNLDED_STEM = "bundled_program_schema"
4+
SCALAR_TYPE_STEM = "scalar_type"
5+
6+
INPUT_BUNDLED = BUNLDED_STEM + ".fbs"
7+
INPUT_SCALAR_TYPE = "//executorch/schema:scalar_type.fbs"
8+
9+
OUTPUT_BUNDLED_HEADER = BUNLDED_STEM + "_generated.h"
10+
OUTPUT_SCALAR_TYPE_HEADER = SCALAR_TYPE_STEM + "_generated.h"
11+
12+
BUNDLED_GEN_RULE_NAME = "generate_bundled_program"
13+
14+
BUNDLED_LIBRARY_NAME = BUNLDED_STEM + "_fbs"
15+
16+
def _generate_schema_header(rule_name, srcs, headers, default_header):
17+
"""Generate header file given flatbuffer schema
18+
"""
19+
runtime.genrule(
20+
name = rule_name,
21+
srcs = srcs,
22+
# We're only generating a single file, so it seems like we could use
23+
# `out`, but `flatc` takes a directory as a parameter, not a single
24+
# file. Use `outs` so that `${OUT}` is expanded as the containing
25+
# directory instead of the file itself.
26+
outs = {header: [header] for header in headers},
27+
default_outs = [default_header],
28+
cmd = " ".join([
29+
"$(exe {})".format(runtime.external_dep_location("flatc")),
30+
"--cpp",
31+
"--cpp-std c++11",
32+
"--gen-mutable",
33+
"--scoped-enums",
34+
"-o ${OUT}",
35+
"${SRCS}",
36+
# Let our infra know that the file was generated.
37+
" ".join(["&& echo // @" + "generated >> ${OUT}/" + header for header in headers]),
38+
]),
39+
visibility = [], # Private
40+
)
41+
42+
def define_common_targets():
43+
"""Defines targets that should be shared between fbcode and xplat.
44+
45+
The directory containing this targets.bzl file should also contain both
46+
TARGETS and BUCK files that call this function.
47+
"""
48+
49+
runtime.export_file(
50+
name = INPUT_BUNDLED,
51+
visibility = [
52+
"//executorch/sdk/bundled_program/serialize/...",
53+
],
54+
)
55+
56+
_generate_schema_header(
57+
BUNDLED_GEN_RULE_NAME,
58+
[INPUT_BUNDLED, INPUT_SCALAR_TYPE],
59+
[OUTPUT_BUNDLED_HEADER, OUTPUT_SCALAR_TYPE_HEADER],
60+
OUTPUT_BUNDLED_HEADER,
61+
)
62+
63+
# Header-only library target with the generate bundled program schema header.
64+
runtime.cxx_library(
65+
name = BUNDLED_LIBRARY_NAME,
66+
srcs = [],
67+
visibility = [
68+
"//executorch/sdk/bundled_program/...",
69+
"//executorch/extension/pybindings/...",
70+
],
71+
exported_headers = {
72+
OUTPUT_BUNDLED_HEADER: ":{}[{}]".format(BUNDLED_GEN_RULE_NAME, OUTPUT_BUNDLED_HEADER),
73+
OUTPUT_SCALAR_TYPE_HEADER: ":{}[{}]".format(BUNDLED_GEN_RULE_NAME, OUTPUT_SCALAR_TYPE_HEADER),
74+
},
75+
exported_external_deps = ["flatbuffers-api"],
76+
)

‎sdk/bundled_program/serialize/TARGETS

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,8 @@ runtime.python_library(
1010
"__init__.py",
1111
],
1212
resources = {
13-
"//executorch/schema:bundled_program_schema.fbs": "bundled_program_schema.fbs",
1413
"//executorch/schema:scalar_type.fbs": "scalar_type.fbs",
14+
"//executorch/sdk/bundled_program/schema:bundled_program_schema.fbs": "bundled_program_schema.fbs",
1515
},
1616
# Currently serialization API should only be used in some dedicated targets,
1717
# to avoid ODR violation when linking with another Flatbuffers library.
@@ -32,6 +32,6 @@ runtime.python_library(
3232
deps = [
3333
"fbsource//third-party/pypi/setuptools:setuptools",
3434
"//executorch/exir/_serialize:lib",
35-
"//executorch/sdk/bundled_program:schema",
35+
"//executorch/sdk/bundled_program/schema:bundled_program_schema_py",
3636
],
3737
)

‎sdk/bundled_program/serialize/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717

1818
from executorch.exir._serialize._dataclass import _DataclassEncoder, _json_to_dataclass
1919
from executorch.exir._serialize._flatbuffer import _flatc_compile, _flatc_decompile
20-
from executorch.sdk.bundled_program.schema import BundledProgram
20+
from executorch.sdk.bundled_program.schema.bundled_program_schema import BundledProgram
2121

2222
# The prefix of schema files used for bundled program
2323
BUNDLED_PROGRAM_SCHEMA_NAME = "bundled_program_schema"

‎sdk/bundled_program/tests/TARGETS

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,9 +29,9 @@ python_unittest(
2929
":lib",
3030
"//caffe2:torch",
3131
"//executorch/exir/_serialize:lib",
32-
"//executorch/sdk/bundled_program:schema",
3332
"//executorch/sdk/bundled_program/bundler:config",
3433
"//executorch/sdk/bundled_program/bundler:core",
34+
"//executorch/sdk/bundled_program/schema:bundled_program_schema_py",
3535
],
3636
)
3737

‎sdk/bundled_program/tests/test_bundle_data.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
import unittest
1010
from typing import List
1111

12-
import executorch.sdk.bundled_program.schema as bp_schema
12+
import executorch.sdk.bundled_program.schema.bundled_program_schema as bp_schema
1313

1414
import torch
1515
from executorch.exir._serialize import _serialize_pte_binary

0 commit comments

Comments
 (0)