Move old ETDump schema to fb specific folder

sdk/etdump/TARGETS

@@ -5,19 +5,6 @@ oncall("executorch")
 
 define_common_targets()
 
-runtime.python_library(
-    name = "schema",
-    srcs = [
-        "schema.py",
-    ],
-    visibility = [
-        "//executorch/sdk/...",
-    ],
-    deps = [
-        "//executorch/exir:scalar_type",
-    ],
-)
-
 runtime.python_library(
     name = "schema_flatcc",
     srcs = [
@@ -38,15 +25,13 @@ runtime.python_library(
     ],
     resources = {
         "//executorch/schema:scalar_type.fbs": "scalar_type.fbs",
-        "//executorch/sdk/etdump:etdump_schema.fbs": "etdump_schema.fbs",
         "//executorch/sdk/etdump:etdump_schema_flatcc.fbs": "etdump_schema_flatcc.fbs",
     },
     visibility = [
         "//executorch/sdk/...",
     ],
     deps = [
         "fbsource//third-party/pypi/setuptools:setuptools",
-        ":schema",
         ":schema_flatcc",
         "//executorch/exir/_serialize:lib",
     ],

sdk/etdump/fb/TARGETS

@@ -0,0 +1,38 @@
+load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
+load(":targets.bzl", "define_common_targets")
+
+oncall("executorch")
+
+define_common_targets()
+
+runtime.python_library(
+    name = "schema",
+    srcs = [
+        "schema.py",
+    ],
+    visibility = [
+        "//executorch/sdk/...",
+    ],
+    deps = [
+        "//executorch/exir:scalar_type",
+    ],
+)
+
+runtime.python_library(
+    name = "serialize",
+    srcs = [
+        "serialize.py",
+    ],
+    resources = {
+        "//executorch/schema:scalar_type.fbs": "scalar_type.fbs",
+        "//executorch/sdk/etdump/fb:etdump_schema.fbs": "etdump_schema.fbs",
+    },
+    visibility = [
+        "//executorch/sdk/...",
+    ],
+    deps = [
+        "fbsource//third-party/pypi/setuptools:setuptools",
+        ":schema",
+        "//executorch/exir/_serialize:lib",
+    ],
+)

sdk/etdump/etdump.cpp → sdk/etdump/fb/etdump.cpp

@@ -1,13 +1,7 @@
-/*
- * Copyright (c) Meta Platforms, Inc. and affiliates.
- * All rights reserved.
- *
- * This source code is licensed under the BSD-style license found in the
- * LICENSE file in the root directory of this source tree.
- */
-
-#include <executorch/sdk/etdump/etdump.h>
-#include <executorch/sdk/etdump/etdump_gen.h>
+// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary.
+
+#include <executorch/sdk/etdump/fb/etdump.h>
+#include <executorch/sdk/etdump/fb/etdump_gen.h>
 
 using namespace torch::executor;
 

sdk/etdump/etdump.h → sdk/etdump/fb/etdump.h

@@ -1,10 +1,4 @@
-/*
- * Copyright (c) Meta Platforms, Inc. and affiliates.
- * All rights reserved.
- *
- * This source code is licensed under the BSD-style license found in the
- * LICENSE file in the root directory of this source tree.
- */
+// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary.
 
 #pragma once
 

sdk/etdump/etdump_gen.cpp → sdk/etdump/fb/etdump_gen.cpp

@@ -1,15 +1,9 @@
-/*
- * Copyright (c) Meta Platforms, Inc. and affiliates.
- * All rights reserved.
- *
- * This source code is licensed under the BSD-style license found in the
- * LICENSE file in the root directory of this source tree.
- */
+// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary.
 
 #include <executorch/runtime/core/memory_allocator.h>
 #include <executorch/runtime/platform/assert.h>
 #include <executorch/runtime/platform/profiler.h>
-#include <executorch/sdk/etdump/etdump_gen.h>
+#include <executorch/sdk/etdump/fb/etdump_gen.h>
 
 using namespace torch::executor;
 

sdk/etdump/etdump_gen.h → sdk/etdump/fb/etdump_gen.h

@@ -1,15 +1,9 @@
-/*
- * Copyright (c) Meta Platforms, Inc. and affiliates.
- * All rights reserved.
- *
- * This source code is licensed under the BSD-style license found in the
- * LICENSE file in the root directory of this source tree.
- */
+// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary.
 
 #pragma once
 
 #include <executorch/runtime/core/memory_allocator.h>
-#include "executorch/sdk/etdump/etdump_schema_generated.h"
+#include "executorch/sdk/etdump/fb/etdump_schema_generated.h"
 
 typedef flatbuffers::FlatBufferBuilder* ETDBuilder;
 

sdk/etdump/fb/scalar_type.fbs

@@ -0,0 +1,32 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+//
+// See README.md before modifying this file.
+//
+
+namespace executorch_flatbuffer;
+
+// The scalar data type.
+// Must match executorch/runtime/core/portable_type/tensor_impl.h
+enum ScalarType : byte {
+  BYTE = 0,
+  CHAR = 1,
+  SHORT = 2,
+  INT = 3,
+  LONG = 4,
+  FLOAT = 6,
+  DOUBLE = 7,
+  BOOL = 11,
+  // TODO(jakeszwe): Verify these are unused and then remove support
+  QINT8 = 12,
+  QUINT8 = 13,
+  QINT32 = 14,
+  QUINT4X2 = 16,
+  QUINT2X4 = 17,
+  // Types currently not implemented.
+  // Half = 5,
+  // COMPLEXHALF = 8,
+  // COMPLEXFLOAT = 9,
+  // COMPLEXDOUBLE = 10,
+  // BFLOAT16 = 15,
+}

sdk/etdump/schema.py → sdk/etdump/fb/schema.py

@@ -1,8 +1,4 @@
-# Copyright (c) Meta Platforms, Inc. and affiliates.
-# All rights reserved.
-#
-# This source code is licensed under the BSD-style license found in the
-# LICENSE file in the root directory of this source tree.
+# (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary.
 
 # pyre-strict
 """

sdk/etdump/fb/serialize.py

@@ -0,0 +1,94 @@
+# (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary.
+
+# pyre-strict
+
+import json
+import os
+import tempfile
+
+import pkg_resources
+
+from executorch.exir._serialize._dataclass import _DataclassEncoder, _json_to_dataclass
+
+from executorch.exir._serialize._flatbuffer import _flatc_compile, _flatc_decompile
+from executorch.sdk.etdump.fb.schema import ETDump
+
+# The prefix of schema files used for etdump
+ETDUMP_SCHEMA_NAME = "etdump_schema"
+SCALAR_TYPE_SCHEMA_NAME = "scalar_type"
+
+
+def _write_schema(d: str, schema_name: str) -> None:
+    schema_path = os.path.join(d, "{}.fbs".format(schema_name))
+    with open(schema_path, "wb") as schema_file:
+        schema_file.write(
+            pkg_resources.resource_string(__name__, "{}.fbs".format(schema_name))
+        )
+
+
+def _serialize_from_etdump_to_json(etdump: ETDump) -> str:
+    return json.dumps(etdump, cls=_DataclassEncoder, indent=4)
+
+
+# from json to etdump
+def _deserialize_from_json_to_etdump(etdump_json: bytes) -> ETDump:
+    etdump_json = json.loads(etdump_json)
+    return _json_to_dataclass(etdump_json, ETDump)
+
+
+def _convert_to_flatbuffer(etdump_json: str) -> bytes:
+    with tempfile.TemporaryDirectory() as d:
+        # load given and common schema
+        _write_schema(d, ETDUMP_SCHEMA_NAME)
+        _write_schema(d, SCALAR_TYPE_SCHEMA_NAME)
+
+        schema_path = os.path.join(d, "{}.fbs".format(ETDUMP_SCHEMA_NAME))
+        json_path = os.path.join(d, "{}.json".format(ETDUMP_SCHEMA_NAME))
+        with open(json_path, "wb") as json_file:
+            json_file.write(etdump_json.encode("ascii"))
+
+        _flatc_compile(d, schema_path, json_path)
+        output_path = os.path.join(d, "{}.etdp".format(ETDUMP_SCHEMA_NAME))
+        with open(output_path, "rb") as output_file:
+            return output_file.read()
+
+
+def _convert_from_flatbuffer(etdump_flatbuffer: bytes) -> bytes:
+    with tempfile.TemporaryDirectory() as d:
+        _write_schema(d, ETDUMP_SCHEMA_NAME)
+        _write_schema(d, SCALAR_TYPE_SCHEMA_NAME)
+
+        schema_path = os.path.join(d, "{}.fbs".format(ETDUMP_SCHEMA_NAME))
+        bin_path = os.path.join(d, "schema.bin")
+        with open(bin_path, "wb") as bin_file:
+            bin_file.write(etdump_flatbuffer)
+        _flatc_decompile(d, schema_path, bin_path)
+        output_path = os.path.join(d, "schema.json")
+        with open(output_path, "rb") as output_file:
+            return output_file.read()
+
+
+def serialize_to_etdump(
+    etdump: ETDump,
+) -> bytes:
+    """
+    Given an ETdump python object this function will return a serialized object
+    that can then be written to a file.
+    Args:
+        etdump: ETDump python object that the user wants to serialize.
+    Returns:
+        Serialized etdump binary blob.
+    """
+    return _convert_to_flatbuffer(_serialize_from_etdump_to_json(etdump))
+
+
+def deserialize_from_etdump(data: bytes) -> ETDump:
+    """
+    Given an etdump binary blob this function will deserialize it and return
+    the python object representation of etdump.
+    Args:
+        data: Serialized etdump binary blob.
+    Returns:
+        Deserialized ETDump python object.
+    """
+    return _deserialize_from_json_to_etdump(_convert_from_flatbuffer(data))

sdk/etdump/fb/targets.bzl

@@ -0,0 +1,95 @@
+load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
+
+ETDUMP_STEM = "etdump_schema"
+ETDUMP_SCHEMA = ETDUMP_STEM + ".fbs"
+ETDUMP_GEN_RULE_NAME = "generate_etdump"
+ETDUMP_LIBRARY_NAME = ETDUMP_STEM
+
+SCALAR_TYPE_STEM = "scalar_type"
+SCALAR_TYPE = SCALAR_TYPE_STEM + ".fbs"
+
+# flatbuffers:flatc
+
+ETDUMP_SCHEMA_HEADER = ETDUMP_STEM + "_generated.h"
+OUTPUT_SCALAR_TYPE_HEADER = SCALAR_TYPE_STEM + "_generated.h"
+
+def generate_schema_header(rule_name, srcs, headers, default_header):
+    """
+    Generate header files for ETDump schema
+    """
+
+    runtime.genrule(
+        name = rule_name,
+        srcs = srcs,
+        outs = {header: [header] for header in headers},
+        default_outs = [default_header],
+        cmd = " ".join([
+            "$(exe {})".format(runtime.external_dep_location("flatc")),
+            "--cpp",
+            "--cpp-std c++11",
+            "--gen-mutable",
+            "--scoped-enums",
+            "-o ${OUT}",
+            "${SRCS}",
+            # Let our infra know that the file was generated.
+            " ".join(["&& echo // @" + "generated >> ${OUT}/" + header for header in headers]),
+        ]),
+    )
+
+def define_common_targets():
+    """Defines targets that should be shared between fbcode and xplat.
+
+    The directory containing this targets.bzl file should also contain both
+    TARGETS and BUCK files that call this function.
+    """
+    runtime.export_file(
+        name = ETDUMP_SCHEMA,
+        visibility = ["//executorch/..."],
+    )
+
+    generate_schema_header(
+        ETDUMP_GEN_RULE_NAME,
+        [ETDUMP_SCHEMA, SCALAR_TYPE],
+        [ETDUMP_SCHEMA_HEADER, OUTPUT_SCALAR_TYPE_HEADER],
+        ETDUMP_SCHEMA_HEADER,
+    )
+
+    runtime.cxx_library(
+        name = ETDUMP_LIBRARY_NAME,
+        srcs = [],
+        visibility = ["//executorch/..."],
+        exported_headers = {
+            ETDUMP_SCHEMA_HEADER: ":{}[{}]".format(ETDUMP_GEN_RULE_NAME, ETDUMP_SCHEMA_HEADER),
+            OUTPUT_SCALAR_TYPE_HEADER: ":{}[{}]".format(ETDUMP_GEN_RULE_NAME, OUTPUT_SCALAR_TYPE_HEADER),
+        },
+        exported_external_deps = ["flatbuffers-api"],
+    )
+
+    runtime.cxx_library(
+        name = "etdump",
+        srcs = ["etdump.cpp"],
+        exported_headers = ["etdump.h"],
+        deps = [
+            ":etdump_gen",
+            "//executorch/runtime/core:core",
+        ],
+        visibility = [
+            "//executorch/...",
+            "@EXECUTORCH_CLIENTS",
+        ],
+    )
+
+    runtime.cxx_library(
+        name = "etdump_gen",
+        srcs = ["etdump_gen.cpp"],
+        exported_headers = ["etdump_gen.h"],
+        deps = [],
+        exported_deps = [
+            ":etdump_schema",
+            "//executorch/runtime/platform:platform",
+            "//executorch/runtime/core:memory_allocator",
+        ],
+        visibility = [
+            "//executorch/...",
+        ],
+    )

sdk/etdump/tests/ETDumpGenTest.cpp → sdk/etdump/fb/tests/ETDumpGenTest.cpp

@@ -1,18 +1,12 @@
-/*
- * Copyright (c) Meta Platforms, Inc. and affiliates.
- * All rights reserved.
- *
- * This source code is licensed under the BSD-style license found in the
- * LICENSE file in the root directory of this source tree.
- */
+// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary.
 
 #include <gtest/gtest.h>
 
 #include <executorch/runtime/core/memory_allocator.h>
 #include <executorch/runtime/platform/profiler.h>
 #include <executorch/runtime/platform/runtime.h>
-#include <executorch/sdk/etdump/etdump_gen.h>
-#include "executorch/sdk/etdump/etdump_schema_generated.h"
+#include <executorch/sdk/etdump/fb/etdump_gen.h>
+#include "executorch/sdk/etdump/fb/etdump_schema_generated.h"
 
 using namespace etdump;