Skip to content

Commit 958c2f5

Browse files
tarun292facebook-github-bot
tarun292
authored and
facebook-github-bot
committed
Move old ETDump schema to fb specific folder (#716)
Summary: The old ETDump schema is not relevant for OSS. Moving it to a fb specific folder, which will later be deprecated. Reviewed By: Jack-Khuu Differential Revision: D50072259
1 parent ca6ece1 commit 958c2f5

20 files changed

+414
-309
lines changed

‎sdk/etdump/TARGETS

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -5,19 +5,6 @@ oncall("executorch")
55

66
define_common_targets()
77

8-
runtime.python_library(
9-
name = "schema",
10-
srcs = [
11-
"schema.py",
12-
],
13-
visibility = [
14-
"//executorch/sdk/...",
15-
],
16-
deps = [
17-
"//executorch/exir:scalar_type",
18-
],
19-
)
20-
218
runtime.python_library(
229
name = "schema_flatcc",
2310
srcs = [
@@ -38,15 +25,13 @@ runtime.python_library(
3825
],
3926
resources = {
4027
"//executorch/schema:scalar_type.fbs": "scalar_type.fbs",
41-
"//executorch/sdk/etdump:etdump_schema.fbs": "etdump_schema.fbs",
4228
"//executorch/sdk/etdump:etdump_schema_flatcc.fbs": "etdump_schema_flatcc.fbs",
4329
},
4430
visibility = [
4531
"//executorch/sdk/...",
4632
],
4733
deps = [
4834
"fbsource//third-party/pypi/setuptools:setuptools",
49-
":schema",
5035
":schema_flatcc",
5136
"//executorch/exir/_serialize:lib",
5237
],

‎sdk/etdump/fb/TARGETS

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
2+
load(":targets.bzl", "define_common_targets")
3+
4+
oncall("executorch")
5+
6+
define_common_targets()
7+
8+
runtime.python_library(
9+
name = "schema",
10+
srcs = [
11+
"schema.py",
12+
],
13+
visibility = [
14+
"//executorch/sdk/...",
15+
],
16+
deps = [
17+
"//executorch/exir:scalar_type",
18+
],
19+
)
20+
21+
runtime.python_library(
22+
name = "serialize",
23+
srcs = [
24+
"serialize.py",
25+
],
26+
resources = {
27+
"//executorch/schema:scalar_type.fbs": "scalar_type.fbs",
28+
"//executorch/sdk/etdump/fb:etdump_schema.fbs": "etdump_schema.fbs",
29+
},
30+
visibility = [
31+
"//executorch/sdk/...",
32+
],
33+
deps = [
34+
"fbsource//third-party/pypi/setuptools:setuptools",
35+
":schema",
36+
"//executorch/exir/_serialize:lib",
37+
],
38+
)

‎sdk/etdump/etdump.cpp renamed to ‎sdk/etdump/fb/etdump.cpp

Lines changed: 4 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,7 @@
1-
/*
2-
* Copyright (c) Meta Platforms, Inc. and affiliates.
3-
* All rights reserved.
4-
*
5-
* This source code is licensed under the BSD-style license found in the
6-
* LICENSE file in the root directory of this source tree.
7-
*/
8-
9-
#include <executorch/sdk/etdump/etdump.h>
10-
#include <executorch/sdk/etdump/etdump_gen.h>
1+
// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary.
2+
3+
#include <executorch/sdk/etdump/fb/etdump.h>
4+
#include <executorch/sdk/etdump/fb/etdump_gen.h>
115

126
using namespace torch::executor;
137

‎sdk/etdump/etdump.h renamed to ‎sdk/etdump/fb/etdump.h

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,4 @@
1-
/*
2-
* Copyright (c) Meta Platforms, Inc. and affiliates.
3-
* All rights reserved.
4-
*
5-
* This source code is licensed under the BSD-style license found in the
6-
* LICENSE file in the root directory of this source tree.
7-
*/
1+
// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary.
82

93
#pragma once
104

‎sdk/etdump/etdump_gen.cpp renamed to ‎sdk/etdump/fb/etdump_gen.cpp

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,9 @@
1-
/*
2-
* Copyright (c) Meta Platforms, Inc. and affiliates.
3-
* All rights reserved.
4-
*
5-
* This source code is licensed under the BSD-style license found in the
6-
* LICENSE file in the root directory of this source tree.
7-
*/
1+
// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary.
82

93
#include <executorch/runtime/core/memory_allocator.h>
104
#include <executorch/runtime/platform/assert.h>
115
#include <executorch/runtime/platform/profiler.h>
12-
#include <executorch/sdk/etdump/etdump_gen.h>
6+
#include <executorch/sdk/etdump/fb/etdump_gen.h>
137

148
using namespace torch::executor;
159

‎sdk/etdump/etdump_gen.h renamed to ‎sdk/etdump/fb/etdump_gen.h

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,9 @@
1-
/*
2-
* Copyright (c) Meta Platforms, Inc. and affiliates.
3-
* All rights reserved.
4-
*
5-
* This source code is licensed under the BSD-style license found in the
6-
* LICENSE file in the root directory of this source tree.
7-
*/
1+
// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary.
82

93
#pragma once
104

115
#include <executorch/runtime/core/memory_allocator.h>
12-
#include "executorch/sdk/etdump/etdump_schema_generated.h"
6+
#include "executorch/sdk/etdump/fb/etdump_schema_generated.h"
137

148
typedef flatbuffers::FlatBufferBuilder* ETDBuilder;
159

‎sdk/etdump/etdump_schema.fbs renamed to ‎sdk/etdump/fb/etdump_schema.fbs

File renamed without changes.

‎sdk/etdump/fb/scalar_type.fbs

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
// Copyright (c) Meta Platforms, Inc. and affiliates.
2+
3+
//
4+
// See README.md before modifying this file.
5+
//
6+
7+
namespace executorch_flatbuffer;
8+
9+
// The scalar data type.
10+
// Must match executorch/runtime/core/portable_type/tensor_impl.h
11+
enum ScalarType : byte {
12+
BYTE = 0,
13+
CHAR = 1,
14+
SHORT = 2,
15+
INT = 3,
16+
LONG = 4,
17+
FLOAT = 6,
18+
DOUBLE = 7,
19+
BOOL = 11,
20+
// TODO(jakeszwe): Verify these are unused and then remove support
21+
QINT8 = 12,
22+
QUINT8 = 13,
23+
QINT32 = 14,
24+
QUINT4X2 = 16,
25+
QUINT2X4 = 17,
26+
// Types currently not implemented.
27+
// Half = 5,
28+
// COMPLEXHALF = 8,
29+
// COMPLEXFLOAT = 9,
30+
// COMPLEXDOUBLE = 10,
31+
// BFLOAT16 = 15,
32+
}

‎sdk/etdump/schema.py renamed to ‎sdk/etdump/fb/schema.py

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,4 @@
1-
# Copyright (c) Meta Platforms, Inc. and affiliates.
2-
# All rights reserved.
3-
#
4-
# This source code is licensed under the BSD-style license found in the
5-
# LICENSE file in the root directory of this source tree.
1+
# (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary.
62

73
# pyre-strict
84
"""

‎sdk/etdump/fb/serialize.py

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
# (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary.
2+
3+
# pyre-strict
4+
5+
import json
6+
import os
7+
import tempfile
8+
9+
import pkg_resources
10+
11+
from executorch.exir._serialize._dataclass import _DataclassEncoder, _json_to_dataclass
12+
13+
from executorch.exir._serialize._flatbuffer import _flatc_compile, _flatc_decompile
14+
from executorch.sdk.etdump.fb.schema import ETDump
15+
16+
# The prefix of schema files used for etdump
17+
ETDUMP_SCHEMA_NAME = "etdump_schema"
18+
SCALAR_TYPE_SCHEMA_NAME = "scalar_type"
19+
20+
21+
def _write_schema(d: str, schema_name: str) -> None:
22+
schema_path = os.path.join(d, "{}.fbs".format(schema_name))
23+
with open(schema_path, "wb") as schema_file:
24+
schema_file.write(
25+
pkg_resources.resource_string(__name__, "{}.fbs".format(schema_name))
26+
)
27+
28+
29+
def _serialize_from_etdump_to_json(etdump: ETDump) -> str:
30+
return json.dumps(etdump, cls=_DataclassEncoder, indent=4)
31+
32+
33+
# from json to etdump
34+
def _deserialize_from_json_to_etdump(etdump_json: bytes) -> ETDump:
35+
etdump_json = json.loads(etdump_json)
36+
return _json_to_dataclass(etdump_json, ETDump)
37+
38+
39+
def _convert_to_flatbuffer(etdump_json: str) -> bytes:
40+
with tempfile.TemporaryDirectory() as d:
41+
# load given and common schema
42+
_write_schema(d, ETDUMP_SCHEMA_NAME)
43+
_write_schema(d, SCALAR_TYPE_SCHEMA_NAME)
44+
45+
schema_path = os.path.join(d, "{}.fbs".format(ETDUMP_SCHEMA_NAME))
46+
json_path = os.path.join(d, "{}.json".format(ETDUMP_SCHEMA_NAME))
47+
with open(json_path, "wb") as json_file:
48+
json_file.write(etdump_json.encode("ascii"))
49+
50+
_flatc_compile(d, schema_path, json_path)
51+
output_path = os.path.join(d, "{}.etdp".format(ETDUMP_SCHEMA_NAME))
52+
with open(output_path, "rb") as output_file:
53+
return output_file.read()
54+
55+
56+
def _convert_from_flatbuffer(etdump_flatbuffer: bytes) -> bytes:
57+
with tempfile.TemporaryDirectory() as d:
58+
_write_schema(d, ETDUMP_SCHEMA_NAME)
59+
_write_schema(d, SCALAR_TYPE_SCHEMA_NAME)
60+
61+
schema_path = os.path.join(d, "{}.fbs".format(ETDUMP_SCHEMA_NAME))
62+
bin_path = os.path.join(d, "schema.bin")
63+
with open(bin_path, "wb") as bin_file:
64+
bin_file.write(etdump_flatbuffer)
65+
_flatc_decompile(d, schema_path, bin_path)
66+
output_path = os.path.join(d, "schema.json")
67+
with open(output_path, "rb") as output_file:
68+
return output_file.read()
69+
70+
71+
def serialize_to_etdump(
72+
etdump: ETDump,
73+
) -> bytes:
74+
"""
75+
Given an ETdump python object this function will return a serialized object
76+
that can then be written to a file.
77+
Args:
78+
etdump: ETDump python object that the user wants to serialize.
79+
Returns:
80+
Serialized etdump binary blob.
81+
"""
82+
return _convert_to_flatbuffer(_serialize_from_etdump_to_json(etdump))
83+
84+
85+
def deserialize_from_etdump(data: bytes) -> ETDump:
86+
"""
87+
Given an etdump binary blob this function will deserialize it and return
88+
the python object representation of etdump.
89+
Args:
90+
data: Serialized etdump binary blob.
91+
Returns:
92+
Deserialized ETDump python object.
93+
"""
94+
return _deserialize_from_json_to_etdump(_convert_from_flatbuffer(data))

‎sdk/etdump/fb/targets.bzl

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
2+
3+
ETDUMP_STEM = "etdump_schema"
4+
ETDUMP_SCHEMA = ETDUMP_STEM + ".fbs"
5+
ETDUMP_GEN_RULE_NAME = "generate_etdump"
6+
ETDUMP_LIBRARY_NAME = ETDUMP_STEM
7+
8+
SCALAR_TYPE_STEM = "scalar_type"
9+
SCALAR_TYPE = SCALAR_TYPE_STEM + ".fbs"
10+
11+
# flatbuffers:flatc
12+
13+
ETDUMP_SCHEMA_HEADER = ETDUMP_STEM + "_generated.h"
14+
OUTPUT_SCALAR_TYPE_HEADER = SCALAR_TYPE_STEM + "_generated.h"
15+
16+
def generate_schema_header(rule_name, srcs, headers, default_header):
17+
"""
18+
Generate header files for ETDump schema
19+
"""
20+
21+
runtime.genrule(
22+
name = rule_name,
23+
srcs = srcs,
24+
outs = {header: [header] for header in headers},
25+
default_outs = [default_header],
26+
cmd = " ".join([
27+
"$(exe {})".format(runtime.external_dep_location("flatc")),
28+
"--cpp",
29+
"--cpp-std c++11",
30+
"--gen-mutable",
31+
"--scoped-enums",
32+
"-o ${OUT}",
33+
"${SRCS}",
34+
# Let our infra know that the file was generated.
35+
" ".join(["&& echo // @" + "generated >> ${OUT}/" + header for header in headers]),
36+
]),
37+
)
38+
39+
def define_common_targets():
40+
"""Defines targets that should be shared between fbcode and xplat.
41+
42+
The directory containing this targets.bzl file should also contain both
43+
TARGETS and BUCK files that call this function.
44+
"""
45+
runtime.export_file(
46+
name = ETDUMP_SCHEMA,
47+
visibility = ["//executorch/..."],
48+
)
49+
50+
generate_schema_header(
51+
ETDUMP_GEN_RULE_NAME,
52+
[ETDUMP_SCHEMA, SCALAR_TYPE],
53+
[ETDUMP_SCHEMA_HEADER, OUTPUT_SCALAR_TYPE_HEADER],
54+
ETDUMP_SCHEMA_HEADER,
55+
)
56+
57+
runtime.cxx_library(
58+
name = ETDUMP_LIBRARY_NAME,
59+
srcs = [],
60+
visibility = ["//executorch/..."],
61+
exported_headers = {
62+
ETDUMP_SCHEMA_HEADER: ":{}[{}]".format(ETDUMP_GEN_RULE_NAME, ETDUMP_SCHEMA_HEADER),
63+
OUTPUT_SCALAR_TYPE_HEADER: ":{}[{}]".format(ETDUMP_GEN_RULE_NAME, OUTPUT_SCALAR_TYPE_HEADER),
64+
},
65+
exported_external_deps = ["flatbuffers-api"],
66+
)
67+
68+
runtime.cxx_library(
69+
name = "etdump",
70+
srcs = ["etdump.cpp"],
71+
exported_headers = ["etdump.h"],
72+
deps = [
73+
":etdump_gen",
74+
"//executorch/runtime/core:core",
75+
],
76+
visibility = [
77+
"//executorch/...",
78+
"@EXECUTORCH_CLIENTS",
79+
],
80+
)
81+
82+
runtime.cxx_library(
83+
name = "etdump_gen",
84+
srcs = ["etdump_gen.cpp"],
85+
exported_headers = ["etdump_gen.h"],
86+
deps = [],
87+
exported_deps = [
88+
":etdump_schema",
89+
"//executorch/runtime/platform:platform",
90+
"//executorch/runtime/core:memory_allocator",
91+
],
92+
visibility = [
93+
"//executorch/...",
94+
],
95+
)

‎sdk/etdump/tests/ETDumpGenTest.cpp renamed to ‎sdk/etdump/fb/tests/ETDumpGenTest.cpp

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,12 @@
1-
/*
2-
* Copyright (c) Meta Platforms, Inc. and affiliates.
3-
* All rights reserved.
4-
*
5-
* This source code is licensed under the BSD-style license found in the
6-
* LICENSE file in the root directory of this source tree.
7-
*/
1+
// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary.
82

93
#include <gtest/gtest.h>
104

115
#include <executorch/runtime/core/memory_allocator.h>
126
#include <executorch/runtime/platform/profiler.h>
137
#include <executorch/runtime/platform/runtime.h>
14-
#include <executorch/sdk/etdump/etdump_gen.h>
15-
#include "executorch/sdk/etdump/etdump_schema_generated.h"
8+
#include <executorch/sdk/etdump/fb/etdump_gen.h>
9+
#include "executorch/sdk/etdump/fb/etdump_schema_generated.h"
1610

1711
using namespace etdump;
1812

0 commit comments

Comments
 (0)