Skip to content

Commit 065a4cf

Browse files
Gasoonjiafacebook-github-bot
Gasoonjia
authored and
facebook-github-bot
committed
move schema files under bundled_program/schema (#1057)
Summary: Pull Request resolved: #1057 Pull Request resolved: #436 This stack focuses on moving all bundled program codes under executorch/sdk/bundled_program. Target and discussions can be found in D49605947. This diff moves all bundled program schemas, both py schema and fbs schema, under bundled_program/schema, and update dependencies. Reviewed By: tarun292 Differential Revision: D50422560 fbshipit-source-id: 10289634f46c657b0f99647659ff0067e06ea314
1 parent beb0aa4 commit 065a4cf

File tree

21 files changed

+237
-75
lines changed

21 files changed

+237
-75
lines changed

‎extension/pybindings/pybindings.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,8 @@
2525
#include <executorch/runtime/platform/assert.h>
2626
#include <executorch/runtime/platform/profiler.h>
2727
#include <executorch/runtime/platform/runtime.h>
28-
#include <executorch/schema/bundled_program_schema_generated.h>
2928
#include <executorch/sdk/bundled_program/bundled_program.h>
29+
#include <executorch/sdk/bundled_program/schema/bundled_program_schema_generated.h>
3030
#include <executorch/util/read_file.h>
3131

3232
#include <ATen/Functions.h>

‎schema/CMakeLists.txt

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,3 @@ generate_program_schema("${common_schema_srcs}" "common_schema")
5656
set(program_schema_srcs program.fbs)
5757
generate_program_schema("${program_schema_srcs}" "program_schema")
5858
add_dependencies(program_schema common_schema)
59-
60-
set(bundled_schema_srcs bundled_program_schema.fbs)
61-
generate_program_schema("${bundled_schema_srcs}" "bundled_schema")
62-
add_dependencies(bundled_schema common_schema)

‎schema/README.md

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,6 @@ The `schema.fbs` file in this directory describes the
22
[Flatbuffers](https://google.github.io/flatbuffers/) schema used to serialize
33
ExecuTorch programs.
44

5-
The `bundled_program_schema.fbs` file is for serializing bundled program. It
6-
bundles the ExecuTorch program, several sets of inputs and referenced outputs,
7-
and other useful info together for verifying the correctness of ExecuTorch program.
8-
95
The `scalar_type.fbs` file contains schema for scalar types, used in both
106
`schema.fbs` and `bundled_program_schema.fbs`.
117

‎schema/targets.bzl

Lines changed: 0 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -2,22 +2,17 @@ load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
22

33
# Construct the input and output file names. All input and output files rely on scalar_type file.
44
PROGRAM_STEM = "program"
5-
BUNLDED_STEM = "bundled_program_schema"
65
SCALAR_TYPE_STEM = "scalar_type"
76

87
INPUT_PROGRAM = PROGRAM_STEM + ".fbs"
9-
INPUT_BUNDLED = BUNLDED_STEM + ".fbs"
108
INPUT_SCALAR_TYPE = SCALAR_TYPE_STEM + ".fbs"
119

1210
OUTPUT_PROGRAM_HEADER = PROGRAM_STEM + "_generated.h"
13-
OUTPUT_BUNDLED_HEADER = BUNLDED_STEM + "_generated.h"
1411
OUTPUT_SCALAR_TYPE_HEADER = SCALAR_TYPE_STEM + "_generated.h"
1512

1613
PROGRAM_GEN_RULE_NAME = "generate_program"
17-
BUNDLED_GEN_RULE_NAME = "generate_bundled_program"
1814

1915
PROGRAM_LIRRARY_NAME = PROGRAM_STEM
20-
BUNDLED_LIBRARY_NAME = BUNLDED_STEM
2116

2217
def _generate_schema_header(rule_name, srcs, headers, default_header):
2318
"""Generate header file given flatbuffer schema
@@ -58,16 +53,9 @@ def define_common_targets():
5853
"//executorch/exir/_serialize/...",
5954
],
6055
)
61-
runtime.export_file(
62-
name = INPUT_BUNDLED,
63-
visibility = [
64-
"//executorch/sdk/bundled_program/serialize/...",
65-
],
66-
)
6756
runtime.export_file(
6857
name = INPUT_SCALAR_TYPE,
6958
visibility = [
70-
"//executorch/sdk/bundled_program/serialize/...",
7159
"//executorch/exir/_serialize/...",
7260
"//executorch/sdk/etdump/...",
7361
],
@@ -80,13 +68,6 @@ def define_common_targets():
8068
OUTPUT_PROGRAM_HEADER,
8169
)
8270

83-
_generate_schema_header(
84-
BUNDLED_GEN_RULE_NAME,
85-
[INPUT_BUNDLED, INPUT_SCALAR_TYPE],
86-
[OUTPUT_BUNDLED_HEADER, OUTPUT_SCALAR_TYPE_HEADER],
87-
OUTPUT_BUNDLED_HEADER,
88-
)
89-
9071
# Header-only library target with the generate executorch program schema header.
9172
runtime.cxx_library(
9273
name = PROGRAM_LIRRARY_NAME,
@@ -95,7 +76,6 @@ def define_common_targets():
9576
# Lock this down as tightly as possible to ensure that flatbuffers
9677
# are an implementation detail. Ideally this list would only include
9778
# //executorch/runtime/executor/...
98-
"//executorch/bundled_program/runtime/...",
9979
"//executorch/codegen/tools/...",
10080
"//executorch/runtime/executor/...",
10181
],
@@ -106,23 +86,6 @@ def define_common_targets():
10686
exported_external_deps = ["flatbuffers-api"],
10787
)
10888

109-
# Header-only library target with the generate bundled program schema header.
110-
runtime.cxx_library(
111-
name = BUNDLED_LIBRARY_NAME,
112-
srcs = [],
113-
visibility = [
114-
"//executorch/sdk/bundled_program/...",
115-
"//executorch/extension/pybindings/...",
116-
"//executorch/examples/bundled_executor_runner/...",
117-
"//executorch/util/...", # bundled_program_verification
118-
],
119-
exported_headers = {
120-
OUTPUT_BUNDLED_HEADER: ":{}[{}]".format(BUNDLED_GEN_RULE_NAME, OUTPUT_BUNDLED_HEADER),
121-
OUTPUT_SCALAR_TYPE_HEADER: ":{}[{}]".format(PROGRAM_GEN_RULE_NAME, OUTPUT_SCALAR_TYPE_HEADER),
122-
},
123-
exported_external_deps = ["flatbuffers-api"],
124-
)
125-
12689
runtime.cxx_library(
12790
name = "extended_header",
12891
srcs = ["extended_header.cpp"],

‎sdk/CMakeLists.txt

Lines changed: 38 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,14 @@ set(_etdump_schema__srcs
3737
${CMAKE_CURRENT_SOURCE_DIR}/etdump/etdump_schema_flatcc.fbs
3838
${CMAKE_CURRENT_SOURCE_DIR}/etdump/scalar_type.fbs)
3939

40+
41+
set (_bundled_program_schema_dir
42+
${CMAKE_CURRENT_SOURCE_DIR}/bundled_program/schema)
43+
44+
set(_bundled_program_schema__srcs
45+
${_bundled_program_schema_dir}/bundled_program_schema.fbs
46+
${_bundled_program_schema_dir}/scalar_type.fbs)
47+
4048
add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../third-party/flatcc
4149
${CMAKE_BINARY_DIR}/third-party/flatcc)
4250

@@ -63,13 +71,27 @@ foreach(fbs_file ${_etdump_schema__srcs})
6371
"${_program_schema__include_dir}/${generated}")
6472
endforeach()
6573

74+
set(_bundled_program_schema__outputs)
75+
foreach(fbs_file ${_bundled_program_schema__srcs})
76+
string(REGEX REPLACE "[.]fbs$" "_generated.h" generated "${fbs_file}")
77+
list(APPEND _bundled_program_schema__outputs
78+
"${_program_schema__include_dir}/${generated}")
79+
endforeach()
80+
6681
add_library(etdump_schema INTERFACE ${_etdump_schema__outputs})
82+
add_library(bundled_program_schema
83+
INTERFACE ${_bundled_program_schema__outputs})
84+
6785
# Ensure the host tool is built before the main project
6886
add_dependencies(etdump_schema flatcc_cli)
6987

7088
message("Debug _etdump_schema__outputs: ${_etdump_schema__outputs}")
89+
message("Debug _bundled_program_schema__outputs:\
90+
${_bundled_program_schema__outputs}")
7191

7292
file(MAKE_DIRECTORY ${_program_schema__include_dir}/executorch/sdk/etdump)
93+
file(MAKE_DIRECTORY
94+
${_program_schema__include_dir}/executorch/sdk/bundled_program)
7395

7496
add_custom_command(
7597
OUTPUT ${_etdump_schema__outputs}
@@ -82,12 +104,23 @@ add_custom_command(
82104
COMMENT "Generating etdump headers"
83105
VERBATIM)
84106

85-
add_library(etdump ${CMAKE_CURRENT_SOURCE_DIR}/etdump/etdump_flatcc.cpp)
86-
target_link_libraries(
87-
etdump
88-
PUBLIC etdump_schema
89-
PRIVATE executorch)
107+
add_custom_command(
108+
OUTPUT ${_bundled_program_schema__outputs}
109+
COMMAND
110+
${CMAKE_SOURCE_DIR}/third-party/flatcc/bin/flatcc -cwr -o
111+
${_program_schema__include_dir}/executorch/sdk/bundled_program
112+
${_bundled_program_schema__srcs}
113+
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/sdk
114+
DEPENDS flatcc_project
115+
COMMENT "Generating bundled_program headers"
116+
VERBATIM)
90117

91118
target_include_directories(
92119
etdump PUBLIC ${_program_schema__include_dir}
93120
${CMAKE_SOURCE_DIR}/third-party/flatcc/include)
121+
122+
set_target_properties(bundled_program PROPERTIES LINKER_LANGUAGE CXX)
123+
target_include_directories(
124+
bundled_program
125+
INTERFACE ${_program_schema__include_dir}
126+
${EXECUTORCH_ROOT}/third-party/flatbuffers/include)

‎sdk/bundled_program/TARGETS

Lines changed: 1 addition & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,12 @@ runtime.python_library(
1616
],
1717
deps = [
1818
":config",
19-
":schema",
2019
":version",
2120
"//caffe2:torch",
2221
"//executorch/exir:schema",
2322
"//executorch/exir:tensor",
2423
"//executorch/exir/_serialize:lib",
24+
"//executorch/sdk/bundled_program/schema:bundled_program_schema_py",
2525
],
2626
)
2727

@@ -40,19 +40,6 @@ runtime.python_library(
4040
],
4141
)
4242

43-
runtime.python_library(
44-
name = "schema",
45-
srcs = [
46-
"schema.py",
47-
],
48-
visibility = [
49-
"//executorch/sdk/...",
50-
],
51-
deps = [
52-
"//executorch/exir:scalar_type",
53-
],
54-
)
55-
5643
runtime.python_library(
5744
name = "version",
5845
srcs = [

‎sdk/bundled_program/bundled_program.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
#include <executorch/runtime/core/memory_allocator.h>
2121
#include <executorch/runtime/executor/method.h>
2222
#include <executorch/runtime/platform/log.h>
23-
#include <executorch/schema/bundled_program_schema_generated.h>
23+
#include <executorch/sdk/bundled_program/schema/bundled_program_schema_generated.h>
2424

2525
namespace torch {
2626
namespace executor {

‎sdk/bundled_program/schema/README.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
The `bundled_program_schema.fbs` file is for serializing bundled program. It
2+
bundles the ExecuTorch program, several sets of inputs and referenced outputs,
3+
and other useful info together for verifying the correctness of ExecuTorch program.
4+
5+
## Rules to ensure forward/backward compatibility
6+
Please check the rules in [here](../../../schema/README.md) for more info.

‎sdk/bundled_program/schema/TARGETS

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# Any targets that should be shared between fbcode and xplat must be defined in
2+
# targets.bzl. This file can contain fbcode-only targets.
3+
4+
load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
5+
load(":targets.bzl", "define_common_targets")
6+
7+
define_common_targets()
8+
9+
runtime.python_library(
10+
name = "bundled_program_schema_py",
11+
srcs = [
12+
"__init__.py",
13+
"bundled_program_schema.py",
14+
],
15+
visibility = [
16+
"//executorch/sdk/...",
17+
],
18+
deps = [
19+
"//executorch/exir:scalar_type",
20+
],
21+
)

‎sdk/bundled_program/schema/__init__.py

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
# pyre-strict
8+
9+
from .bundled_program_schema import * # noqa

‎schema/bundled_program_schema.fbs renamed to ‎sdk/bundled_program/schema/bundled_program_schema.fbs

File renamed without changes.

‎sdk/bundled_program/schema.py renamed to ‎sdk/bundled_program/schema/bundled_program_schema.py

File renamed without changes.

‎sdk/bundled_program/schema/scalar_type.fbs

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
// Copyright (c) Meta Platforms, Inc. and affiliates.
2+
3+
//
4+
// See README.md before modifying this file.
5+
//
6+
7+
namespace executorch_flatbuffer;
8+
9+
// The scalar data type.
10+
// Must match executorch/runtime/core/portable_type/tensor_impl.h
11+
enum ScalarType : byte {
12+
BYTE = 0,
13+
CHAR = 1,
14+
SHORT = 2,
15+
INT = 3,
16+
LONG = 4,
17+
FLOAT = 6,
18+
DOUBLE = 7,
19+
BOOL = 11,
20+
// TODO(jakeszwe): Verify these are unused and then remove support
21+
QINT8 = 12,
22+
QUINT8 = 13,
23+
QINT32 = 14,
24+
QUINT4X2 = 16,
25+
QUINT2X4 = 17,
26+
// Types currently not implemented.
27+
// Half = 5,
28+
// COMPLEXHALF = 8,
29+
// COMPLEXFLOAT = 9,
30+
// COMPLEXDOUBLE = 10,
31+
// BFLOAT16 = 15,
32+
}

‎sdk/bundled_program/schema/targets.bzl

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
2+
3+
BUNLDED_STEM = "bundled_program_schema"
4+
SCALAR_TYPE_STEM = "scalar_type"
5+
6+
INPUT_BUNDLED = BUNLDED_STEM + ".fbs"
7+
INPUT_SCALAR_TYPE = SCALAR_TYPE_STEM + ".fbs"
8+
9+
OUTPUT_BUNDLED_HEADER = BUNLDED_STEM + "_generated.h"
10+
OUTPUT_SCALAR_TYPE_HEADER = SCALAR_TYPE_STEM + "_generated.h"
11+
12+
BUNDLED_GEN_RULE_NAME = "generate_bundled_program"
13+
14+
BUNDLED_LIBRARY_NAME = BUNLDED_STEM + "_fbs"
15+
16+
def _generate_schema_header(rule_name, srcs, headers, default_header):
17+
"""Generate header file given flatbuffer schema
18+
"""
19+
runtime.genrule(
20+
name = rule_name,
21+
srcs = srcs,
22+
# We're only generating a single file, so it seems like we could use
23+
# `out`, but `flatc` takes a directory as a parameter, not a single
24+
# file. Use `outs` so that `${OUT}` is expanded as the containing
25+
# directory instead of the file itself.
26+
outs = {header: [header] for header in headers},
27+
default_outs = [default_header],
28+
cmd = " ".join([
29+
"$(exe {})".format(runtime.external_dep_location("flatc")),
30+
"--cpp",
31+
"--cpp-std c++11",
32+
"--gen-mutable",
33+
"--scoped-enums",
34+
"-o ${OUT}",
35+
"${SRCS}",
36+
# Let our infra know that the file was generated.
37+
" ".join(["&& echo // @" + "generated >> ${OUT}/" + header for header in headers]),
38+
]),
39+
visibility = [], # Private
40+
)
41+
42+
def define_common_targets():
43+
"""Defines targets that should be shared between fbcode and xplat.
44+
45+
The directory containing this targets.bzl file should also contain both
46+
TARGETS and BUCK files that call this function.
47+
"""
48+
49+
runtime.export_file(
50+
name = INPUT_BUNDLED,
51+
visibility = [
52+
"//executorch/sdk/bundled_program/serialize/...",
53+
],
54+
)
55+
56+
runtime.export_file(
57+
name = INPUT_SCALAR_TYPE,
58+
visibility = [
59+
"//executorch/sdk/bundled_program/serialize/...",
60+
],
61+
)
62+
63+
_generate_schema_header(
64+
BUNDLED_GEN_RULE_NAME,
65+
[INPUT_BUNDLED, INPUT_SCALAR_TYPE],
66+
[OUTPUT_BUNDLED_HEADER, OUTPUT_SCALAR_TYPE_HEADER],
67+
OUTPUT_BUNDLED_HEADER,
68+
)
69+
70+
# Header-only library target with the generate bundled program schema header.
71+
runtime.cxx_library(
72+
name = BUNDLED_LIBRARY_NAME,
73+
srcs = [],
74+
visibility = [
75+
"//executorch/sdk/bundled_program/...",
76+
"//executorch/extension/pybindings/...",
77+
],
78+
exported_headers = {
79+
OUTPUT_BUNDLED_HEADER: ":{}[{}]".format(BUNDLED_GEN_RULE_NAME, OUTPUT_BUNDLED_HEADER),
80+
OUTPUT_SCALAR_TYPE_HEADER: ":{}[{}]".format(BUNDLED_GEN_RULE_NAME, OUTPUT_SCALAR_TYPE_HEADER),
81+
},
82+
exported_external_deps = ["flatbuffers-api"],
83+
)

‎sdk/bundled_program/schema/test/TARGETS

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
# @noautodeps
2+
3+
load("@fbcode_macros//build_defs:python_unittest.bzl", "python_unittest")
4+
5+
oncall("executorch")
6+
7+
python_unittest(
8+
name = "schema",
9+
srcs = [
10+
"test_schema.py",
11+
],
12+
)

0 commit comments

Comments
 (0)