Automatically generate operator tests

backends/vulkan/test/op_tests/TARGETS

@@ -0,0 +1,5 @@
+load(":targets.bzl", "define_common_targets")
+
+oncall("executorch")
+
+define_common_targets(is_fbcode = True)

backends/vulkan/test/op_tests/cases.py

@@ -0,0 +1,72 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+
+from executorch.backends.vulkan.test.op_tests.utils.codegen import VkTestSuite
+
+
+# Prime numbers dim sizes for testing
+XL = 113
+L = 89
+M2 = 41
+M1 = 37
+M = 29
+S2 = 11
+S1 = 7
+S = 5
+XS = 3
+
+
+def get_binary_elementwise_inputs():
+    return VkTestSuite(
+        [
+            ((M1, M2), (M1, M2)),
+            ((M1, M2), (M1, 1), 2.0),
+            ((M1, M2), (1, M2)),
+            ((S, S1, S2), (S, S1, S2)),
+            ((S, S1, S2), (S, S1, 1), 2.0),
+            ((S, S1, S2), (S, 1, S2), 2.0),
+        ]
+    )
+
+
+def get_mm_inputs():
+    test_suite = VkTestSuite(
+        [
+            ((M1, L), (L, M2)),
+            ((S1, S2), (S2, M)),
+        ],
+    )
+    test_suite.prepacked_args = ["mat2"]
+    return test_suite
+
+
+def get_pool2d_inputs():
+    test_suite = VkTestSuite(
+        [
+            ((S, M1, M2), [2, 2], [1, 1], [0, 0], [1, 1]),
+        ]
+    )
+    test_suite.supports["layouts"] = ["api::GPUMemoryLayout::TENSOR_CHANNELS_PACKED"]
+    return test_suite
+
+
+test_suites = {
+    "aten.add.Tensor": get_binary_elementwise_inputs(),
+    "aten.sub.Tensor": get_binary_elementwise_inputs(),
+    "aten.div.Tensor": get_binary_elementwise_inputs(),
+    "aten.mul.Tensor": get_binary_elementwise_inputs(),
+    "aten.mm.default": get_mm_inputs(),
+    "aten.max_pool2d_with_indices.default": get_pool2d_inputs(),
+}
+
+prepacked_args = {"aten.mm.default": {"mat2"}}
+
+support_exceptions = {
+    "aten.max_pool2d_with_indices.default": {
+        "layouts": ["api::GPUMemoryLayout::TENSOR_CHANNELS_PACKED"]
+    },
+}

backends/vulkan/test/op_tests/generate_op_tests.py

@@ -0,0 +1,79 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+import argparse
+import os
+
+from typing import Dict
+
+from executorch.backends.vulkan.test.op_tests.cases import test_suites
+
+from executorch.backends.vulkan.test.op_tests.utils.codegen import VkCppTestFileGen
+from executorch.backends.vulkan.test.op_tests.utils.codegen_base import (
+    TestSuite,
+    TestSuiteGen,
+)
+
+from torchgen.gen import parse_native_yaml, ParsedYaml
+from torchgen.model import DispatchKey, NativeFunction
+
+
+def registry_name(f: NativeFunction) -> str:
+    name = str(f.namespace) + "." + str(f.func.name)
+    if len(f.func.name.overload_name) == 0:
+        name += ".default"
+    return name
+
+
+def construct_f_map(parsed_yaml: ParsedYaml) -> Dict[str, NativeFunction]:
+    f_map: Dict[str, NativeFunction] = {}
+    for f in parsed_yaml.native_functions:
+        f_map[registry_name(f)] = f
+    return f_map
+
+
+def process_test_suites(
+    cpp_generator: VkCppTestFileGen,
+    f_map: Dict[str, NativeFunction],
+    test_suites: Dict[str, TestSuite],
+) -> None:
+    for registry_name, op_test_suite in test_suites.items():
+        f = f_map[registry_name]
+        cpp_generator.add_suite(registry_name, f, op_test_suite)
+
+
+def generate_cpp(
+    native_functions_yaml_path: str, tags_path: str, output_dir: str
+) -> None:
+    output_file = os.path.join(output_dir, "op_tests.cpp")
+    cpp_generator = VkCppTestFileGen(output_file)
+
+    parsed_yaml = parse_native_yaml(native_functions_yaml_path, tags_path)
+    f_map = construct_f_map(parsed_yaml)
+
+    TestSuiteGen.backend_key = parsed_yaml.backend_indices[DispatchKey.CPU]
+
+    process_test_suites(cpp_generator, f_map, test_suites)
+
+    with open(output_file, "w") as file:
+        file.write(cpp_generator.generate_cpp())
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Generate a simple Hello World C++ program."
+    )
+    parser.add_argument(
+        "--aten-yaml-path",
+        help="path to native_functions.yaml file.",
+    )
+    parser.add_argument(
+        "--tags-path",
+        help="Path to tags.yaml. Required by yaml parsing in codegen system.",
+    )
+    parser.add_argument("-o", "--output", help="Output directory", required=True)
+    args = parser.parse_args()
+    generate_cpp(args.aten_yaml_path, args.tags_path, args.output)

backends/vulkan/test/op_tests/targets.bzl

@@ -0,0 +1,77 @@
+load("@fbsource//tools/build_defs:platform_defs.bzl", "ANDROID")
+load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
+
+def define_common_targets(is_fbcode = False):
+    if is_fbcode:
+        return
+
+    runtime.python_library(
+        name = "generate_op_tests_lib",
+        srcs = native.glob(["utils/*.py"]) + [
+            "generate_op_tests.py",
+            "cases.py",
+        ],
+        base_module = "executorch.backends.vulkan.test.op_tests",
+        deps = [
+            "//caffe2/torchgen:torchgen",
+            "fbsource//third-party/pypi/expecttest:expecttest",
+        ],
+    )
+
+    runtime.python_binary(
+        name = "generate_op_tests",
+        main_module = "executorch.backends.vulkan.test.op_tests.generate_op_tests",
+        deps = [
+            ":generate_op_tests_lib",
+        ],
+    )
+
+    aten_src_path = runtime.external_dep_location("aten-src-path")
+    genrule_cmd = [
+        "$(exe :generate_op_tests)",
+        "--tags-path $(location {})/aten/src/ATen/native/tags.yaml".format(aten_src_path),
+        "--aten-yaml-path $(location {})/aten/src/ATen/native/native_functions.yaml".format(aten_src_path),
+        "-o $OUT",
+    ]
+
+    runtime.genrule(
+        name = "generated_op_tests_cpp",
+        outs = {
+            "op_tests.cpp": ["op_tests.cpp"],
+        },
+        cmd = " ".join(genrule_cmd),
+        default_outs = ["."],
+    )
+
+    runtime.cxx_binary(
+        name = "compute_graph_op_tests_bin",
+        srcs = [
+            ":generated_op_tests_cpp[op_tests.cpp]",
+        ],
+        define_static_target = False,
+        deps = [
+            "//third-party/googletest:gtest_main",
+            "//executorch/backends/vulkan:vulkan_graph_runtime",
+            runtime.external_dep_location("libtorch"),
+        ],
+    )
+
+    runtime.cxx_test(
+        name = "compute_graph_op_tests",
+        srcs = [
+            ":generated_op_tests_cpp[op_tests.cpp]",
+        ],
+        contacts = ["[email protected]"],
+        fbandroid_additional_loaded_sonames = [
+            "torch-code-gen",
+            "vulkan_graph_runtime",
+            "vulkan_graph_runtime_shaderlib",
+        ],
+        platforms = [ANDROID],
+        use_instrumentation_test = True,
+        deps = [
+            "//third-party/googletest:gtest_main",
+            "//executorch/backends/vulkan:vulkan_graph_runtime",
+            runtime.external_dep_location("libtorch"),
+        ],
+    )