Skip to content

Commit 218432d

Browse files
Gasoonjiafacebook-github-bot
Gasoonjia
authored and
facebook-github-bot
committed
move schema files under bundled_program/schema (#1057)
Summary: Pull Request resolved: #1057 Pull Request resolved: #436 This stack focuses on moving all bundled program codes under executorch/sdk/bundled_program. Target and discussions can be found in D49605947. This diff moves all bundled program schemas, both py schema and fbs schema, under bundled_program/schema, and update dependencies. Differential Revision: D50422560 fbshipit-source-id: 68453569168b4a6a8495ab1d2e552e2497dd8db7
1 parent 59f015d commit 218432d

File tree

15 files changed

+215
-59
lines changed

15 files changed

+215
-59
lines changed

‎extension/pybindings/pybindings.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,8 @@
2525
#include <executorch/runtime/platform/assert.h>
2626
#include <executorch/runtime/platform/profiler.h>
2727
#include <executorch/runtime/platform/runtime.h>
28-
#include <executorch/schema/bundled_program_schema_generated.h>
2928
#include <executorch/sdk/bundled_program/runtime/bundled_program_verification.h>
29+
#include <executorch/sdk/bundled_program/schema/bundled_program_schema_generated.h>
3030
#include <executorch/util/read_file.h>
3131

3232
#include <ATen/Functions.h>

‎schema/targets.bzl

Lines changed: 2 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -2,22 +2,17 @@ load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
22

33
# Construct the input and output file names. All input and output files rely on scalar_type file.
44
PROGRAM_STEM = "program"
5-
BUNLDED_STEM = "bundled_program_schema"
65
SCALAR_TYPE_STEM = "scalar_type"
76

87
INPUT_PROGRAM = PROGRAM_STEM + ".fbs"
9-
INPUT_BUNDLED = BUNLDED_STEM + ".fbs"
108
INPUT_SCALAR_TYPE = SCALAR_TYPE_STEM + ".fbs"
119

1210
OUTPUT_PROGRAM_HEADER = PROGRAM_STEM + "_generated.h"
13-
OUTPUT_BUNDLED_HEADER = BUNLDED_STEM + "_generated.h"
1411
OUTPUT_SCALAR_TYPE_HEADER = SCALAR_TYPE_STEM + "_generated.h"
1512

1613
PROGRAM_GEN_RULE_NAME = "generate_program"
17-
BUNDLED_GEN_RULE_NAME = "generate_bundled_program"
1814

1915
PROGRAM_LIRRARY_NAME = PROGRAM_STEM
20-
BUNDLED_LIBRARY_NAME = BUNLDED_STEM
2116

2217
def _generate_schema_header(rule_name, srcs, headers, default_header):
2318
"""Generate header file given flatbuffer schema
@@ -58,17 +53,12 @@ def define_common_targets():
5853
"//executorch/exir/_serialize/...",
5954
],
6055
)
61-
runtime.export_file(
62-
name = INPUT_BUNDLED,
63-
visibility = [
64-
"//executorch/sdk/bundled_program/serialize/...",
65-
],
66-
)
6756
runtime.export_file(
6857
name = INPUT_SCALAR_TYPE,
6958
visibility = [
70-
"//executorch/sdk/bundled_program/serialize/...",
7159
"//executorch/exir/_serialize/...",
60+
"//executorch/sdk/bundled_program/schema/...",
61+
"//executorch/sdk/bundled_program/serialize/...",
7262
"//executorch/sdk/etdump/...",
7363
],
7464
)
@@ -80,13 +70,6 @@ def define_common_targets():
8070
OUTPUT_PROGRAM_HEADER,
8171
)
8272

83-
_generate_schema_header(
84-
BUNDLED_GEN_RULE_NAME,
85-
[INPUT_BUNDLED, INPUT_SCALAR_TYPE],
86-
[OUTPUT_BUNDLED_HEADER, OUTPUT_SCALAR_TYPE_HEADER],
87-
OUTPUT_BUNDLED_HEADER,
88-
)
89-
9073
# Header-only library target with the generate executorch program schema header.
9174
runtime.cxx_library(
9275
name = PROGRAM_LIRRARY_NAME,
@@ -95,7 +78,6 @@ def define_common_targets():
9578
# Lock this down as tightly as possible to ensure that flatbuffers
9679
# are an implementation detail. Ideally this list would only include
9780
# //executorch/runtime/executor/...
98-
"//executorch/bundled_program/runtime/...",
9981
"//executorch/codegen/tools/...",
10082
"//executorch/runtime/executor/...",
10183
],
@@ -106,22 +88,6 @@ def define_common_targets():
10688
exported_external_deps = ["flatbuffers-api"],
10789
)
10890

109-
# Header-only library target with the generate bundled program schema header.
110-
runtime.cxx_library(
111-
name = BUNDLED_LIBRARY_NAME,
112-
srcs = [],
113-
visibility = [
114-
"//executorch/sdk/bundled_program/...",
115-
"//executorch/extension/pybindings/...",
116-
"//executorch/examples/bundled_executor_runner/...",
117-
"//executorch/util/...", # bundled_program_verification
118-
],
119-
exported_headers = {
120-
OUTPUT_BUNDLED_HEADER: ":{}[{}]".format(BUNDLED_GEN_RULE_NAME, OUTPUT_BUNDLED_HEADER),
121-
OUTPUT_SCALAR_TYPE_HEADER: ":{}[{}]".format(PROGRAM_GEN_RULE_NAME, OUTPUT_SCALAR_TYPE_HEADER),
122-
},
123-
exported_external_deps = ["flatbuffers-api"],
124-
)
12591

12692
runtime.cxx_library(
12793
name = "extended_header",

‎sdk/bundled_program/TARGETS

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,12 @@ python_library(
99
],
1010
deps = [
1111
":config",
12-
":schema",
1312
":version",
1413
"//caffe2:torch",
1514
"//executorch/exir:schema",
1615
"//executorch/exir:tensor",
1716
"//executorch/exir/_serialize:lib",
17+
"//executorch/sdk/bundled_program/schema:bundled_program_schema_py",
1818
],
1919
)
2020

@@ -29,16 +29,6 @@ python_library(
2929
],
3030
)
3131

32-
python_library(
33-
name = "schema",
34-
srcs = [
35-
"schema.py",
36-
],
37-
deps = [
38-
"//executorch/exir:scalar_type",
39-
],
40-
)
41-
4232
python_library(
4333
name = "version",
4434
srcs = [

‎sdk/bundled_program/runtime/TARGETS

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# Any targets that should be shared between fbcode and xplat must be defined in
2-
# targets.bzl. This file can contain xplat-only targets.
2+
# targets.bzl. This file can contain fbcode-only targets.
33

44
load(":targets.bzl", "define_common_targets")
55

‎sdk/bundled_program/runtime/bundled_program_verification.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
#include <executorch/runtime/core/memory_allocator.h>
2121
#include <executorch/runtime/executor/method.h>
2222
#include <executorch/runtime/platform/log.h>
23-
#include <executorch/schema/bundled_program_schema_generated.h>
23+
#include <executorch/sdk/bundled_program/schema/bundled_program_schema_generated.h>
2424

2525
namespace torch {
2626
namespace executor {

‎sdk/bundled_program/runtime/targets.bzl

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ def define_common_targets():
1919
],
2020
deps = [
2121
"//executorch/runtime/core/exec_aten/util:dim_order_util" + aten_suffix,
22-
"//executorch/schema:bundled_program_schema",
22+
"//executorch/sdk/bundled_program/schema:bundled_program_schema_fbs",
2323
],
2424
exported_deps = [
2525
"//executorch/runtime/core:memory_allocator",

‎sdk/bundled_program/schema/TARGETS

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
# Any targets that should be shared between fbcode and xplat must be defined in
2+
# targets.bzl. This file can contain fbcode-only targets.
3+
4+
load("@fbcode_macros//build_defs:python_library.bzl", "python_library")
5+
load(":targets.bzl", "define_common_targets")
6+
7+
define_common_targets()
8+
9+
python_library(
10+
name = "bundled_program_schema_py",
11+
srcs = [
12+
"__init__.py",
13+
"bundled_program_schema.py",
14+
],
15+
deps = [
16+
"//executorch/exir:scalar_type",
17+
],
18+
)

‎sdk/bundled_program/schema/__init__.py

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
# pyre-strict
8+
9+
from .bundled_program_schema import *

‎sdk/bundled_program/schema/bundled_program_schema.fbs

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
// Copyright (c) Meta Platforms, Inc. and affiliates.
2+
3+
//
4+
// See README.md before modifying this file.
5+
//
6+
7+
include "scalar_type.fbs";
8+
9+
namespace bundled_program_flatbuffer;
10+
11+
// Identifier of a valid bundled program schema.
12+
file_identifier "BP06";
13+
// Extension of written files.
14+
file_extension "bpte";
15+
16+
// Reason for basic struct: union value type can only be table/struct/string
17+
table Int {
18+
int_val:long;
19+
}
20+
21+
table Bool {
22+
bool_val:bool;
23+
}
24+
25+
table Double {
26+
double_val:double;
27+
}
28+
29+
// All information we need to bundle for a tensor EValue input.
30+
table Tensor {
31+
// The scalar type of Tensor
32+
scalar_type: executorch_flatbuffer.ScalarType;
33+
// The target sizes of the tensor.
34+
sizes: [int];
35+
// The contents of the corresponding input tensor.
36+
data: [ubyte] (force_align: 16);
37+
dim_order:[ubyte];
38+
}
39+
40+
union ValueUnion {
41+
Tensor,
42+
Int,
43+
Bool,
44+
Double,
45+
}
46+
47+
// Abstraction for BundledMethodTestCase values
48+
table Value {
49+
val: ValueUnion;
50+
}
51+
52+
// A single test for a method. The provided inputs should produce the
53+
// expected outputs.
54+
table BundledMethodTestCase {
55+
// The inputs to provide to the method. The number and types of inputs must
56+
// match the schema of the method under test.
57+
inputs: [Value];
58+
59+
// The expected outputs generated while running the model in eager mode using
60+
// the inputs provided. Its length should be equal to the length of program
61+
// outputs.
62+
expected_outputs: [Value];
63+
}
64+
65+
// Collection of test cases for a program method.
66+
table BundledMethodTestSuite {
67+
// The name of the method to test; e.g., "forward" for the forward() method
68+
// of an nn.Module. This name match a method defined by the ExecuTorch
69+
// program.
70+
method_name: string;
71+
72+
// Individual test cases for the method.
73+
test_cases: [BundledMethodTestCase];
74+
}
75+
76+
77+
// Executorch program bunlded with data for verification.
78+
table BundledProgram {
79+
// Schema version.
80+
version:uint;
81+
82+
// Test sets to run against the program.
83+
// Each BundledMethodTestSuite should be used for the method of program sharing same name.
84+
method_test_suites: [BundledMethodTestSuite];
85+
86+
// The binary data of a serialized Executorch program.
87+
// The following `force_align` may sliently override any larger force_align
88+
// used in the program. Therefore, to keep the data (including constant
89+
// tensor, delegate data, etc, see schema.fbs for more info) in the
90+
// executorch program keeps the same alignment as original no matter how
91+
// the program schema changes, we need to make the force_align here the max
92+
// one around all kinds of force_align in the current and future program
93+
// schema, so we use the 4096 as the force_align here.
94+
program: [ubyte] (force_align: 4096);
95+
}
96+
97+
root_type BundledProgram;

‎sdk/bundled_program/schema.py renamed to ‎sdk/bundled_program/schema/bundled_program_schema.py

File renamed without changes.

‎sdk/bundled_program/schema/targets.bzl

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
2+
3+
BUNLDED_STEM = "bundled_program_schema"
4+
SCALAR_TYPE_STEM = "scalar_type"
5+
6+
INPUT_BUNDLED = BUNLDED_STEM + ".fbs"
7+
INPUT_SCALAR_TYPE = "//executorch/schema:scalar_type.fbs"
8+
9+
OUTPUT_BUNDLED_HEADER = BUNLDED_STEM + "_generated.h"
10+
OUTPUT_SCALAR_TYPE_HEADER = SCALAR_TYPE_STEM + "_generated.h"
11+
12+
BUNDLED_GEN_RULE_NAME = "generate_bundled_program"
13+
14+
BUNDLED_LIBRARY_NAME = BUNLDED_STEM + "_fbs"
15+
16+
def _generate_schema_header(rule_name, srcs, headers, default_header):
17+
"""Generate header file given flatbuffer schema
18+
"""
19+
runtime.genrule(
20+
name = rule_name,
21+
srcs = srcs,
22+
# We're only generating a single file, so it seems like we could use
23+
# `out`, but `flatc` takes a directory as a parameter, not a single
24+
# file. Use `outs` so that `${OUT}` is expanded as the containing
25+
# directory instead of the file itself.
26+
outs = {header: [header] for header in headers},
27+
default_outs = [default_header],
28+
cmd = " ".join([
29+
"$(exe {})".format(runtime.external_dep_location("flatc")),
30+
"--cpp",
31+
"--cpp-std c++11",
32+
"--gen-mutable",
33+
"--scoped-enums",
34+
"-o ${OUT}",
35+
"${SRCS}",
36+
# Let our infra know that the file was generated.
37+
" ".join(["&& echo // @" + "generated >> ${OUT}/" + header for header in headers]),
38+
]),
39+
visibility = [], # Private
40+
)
41+
42+
def define_common_targets():
43+
"""Defines targets that should be shared between fbcode and xplat.
44+
45+
The directory containing this targets.bzl file should also contain both
46+
TARGETS and BUCK files that call this function.
47+
"""
48+
49+
runtime.export_file(
50+
name = INPUT_BUNDLED,
51+
visibility = [
52+
"//executorch/sdk/bundled_program/serialize/...",
53+
],
54+
)
55+
56+
_generate_schema_header(
57+
BUNDLED_GEN_RULE_NAME,
58+
[INPUT_BUNDLED, INPUT_SCALAR_TYPE],
59+
[OUTPUT_BUNDLED_HEADER, OUTPUT_SCALAR_TYPE_HEADER],
60+
OUTPUT_BUNDLED_HEADER,
61+
)
62+
63+
# Header-only library target with the generate bundled program schema header.
64+
runtime.cxx_library(
65+
name = BUNDLED_LIBRARY_NAME,
66+
srcs = [],
67+
visibility = [
68+
"//executorch/sdk/bundled_program/...",
69+
"//executorch/extension/pybindings/...",
70+
],
71+
exported_headers = {
72+
OUTPUT_BUNDLED_HEADER: ":{}[{}]".format(BUNDLED_GEN_RULE_NAME, OUTPUT_BUNDLED_HEADER),
73+
OUTPUT_SCALAR_TYPE_HEADER: ":{}[{}]".format(BUNDLED_GEN_RULE_NAME, OUTPUT_SCALAR_TYPE_HEADER),
74+
},
75+
exported_external_deps = ["flatbuffers-api"],
76+
)

‎sdk/bundled_program/serialize/TARGETS

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,8 @@ runtime.python_library(
1010
"__init__.py",
1111
],
1212
resources = {
13-
"//executorch/schema:bundled_program_schema.fbs": "bundled_program_schema.fbs",
1413
"//executorch/schema:scalar_type.fbs": "scalar_type.fbs",
14+
"//executorch/sdk/bundled_program/schema:bundled_program_schema.fbs": "bundled_program_schema.fbs",
1515
},
1616
# Currently serialization API should only be used in some dedicated targets,
1717
# to avoid ODR violation when linking with another Flatbuffers library.
@@ -32,6 +32,6 @@ runtime.python_library(
3232
deps = [
3333
"fbsource//third-party/pypi/setuptools:setuptools",
3434
"//executorch/exir/_serialize:lib",
35-
"//executorch/sdk/bundled_program:schema",
35+
"//executorch/sdk/bundled_program/schema:bundled_program_schema_py",
3636
],
3737
)

‎sdk/bundled_program/tests/TARGETS

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ python_unittest(
3131
"//executorch/exir/_serialize:lib",
3232
"//executorch/sdk/bundled_program:config",
3333
"//executorch/sdk/bundled_program:core",
34-
"//executorch/sdk/bundled_program:schema",
34+
"//executorch/sdk/bundled_program/schema:bundled_program_schema_py",
3535
],
3636
)
3737

0 commit comments

Comments
 (0)