Skip to content

Commit b4df9a3

Browse files
larryliu0820facebook-github-bot
authored andcommitted
Error out if registering prim ops
Summary: Using `executorch_ops_check` should error out if the dependencies includes 2 libraries of `prim_op_registry`. For example: The target `//arvr/libraries/wristband/tsn/transformers:TorchstreamTransformerTest` depends on both `prim_ops_registry` and `prim_ops_registry_aten`. BUCK query: ``` buck2 uquery "filter('prim_ops_registry(?:_static|_aten)?$', deps(//arvr/libraries/wristband/tsn/transformers: TorchstreamTransformerTest))" Buck UI: https://www.internalfb.com/buck2/1401c12c-0ef2-4ba8-8d4e-6b86871d708f Network: Up: 0B Down: 0B Command: uquery. Time elapsed: 3.6s fbcode//executorch/kernels/prim_ops:prim_ops_registry fbcode//executorch/kernels/prim_ops:prim_ops_registry_aten fbsource//xplat/executorch/kernels/prim_ops:prim_ops_registry fbsource//xplat/executorch/kernels/prim_ops:prim_ops_registry_aten ``` This will cause the error like this: ``` E 00:00:00.032942 executorch:operator_registry.cpp:86] Re-registering aten::sym_size.int, from /data/users/larryliu/fbsource/buck-out/v2/gen/fbsource/cfdc20bd56300721/arvr/libraries/wristband/tsn/transformers/__TorchstreamTransformerTest__/./__TorchstreamTransformerTest__shared_libs_symlink_tree/libexecutorc$ E 00:00:00.033022 executorch:operator_registry.cpp:87] key: (null), is_fallback: true F 00:00:00.033033 executorch:operator_registry.cpp:111] In function register_kernels(), assert failed (false): Kernel registration failed with error 18, see error log for details. ``` To catch issues like this, we should error out when user uses `executorch_ops_check` to debug. ``` executorch_ops_check( name = "my_executorch_test_check", deps = [":TorchstreamTransformerTest"], ) ``` Fixing the internal portion of [#8171](#8171) Differential Revision: D69090850
1 parent 0a936e0 commit b4df9a3

File tree

2 files changed

+35
-4
lines changed

2 files changed

+35
-4
lines changed

codegen/tools/gen_all_oplist.py

Lines changed: 33 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -95,8 +95,35 @@ def main(argv: List[Any]) -> None:
9595
default=False,
9696
required=False,
9797
)
98+
parser.add_argument(
99+
"--DEBUG-ONLY-check-prim-ops",
100+
"--DEBUG_ONLY_check_prim_ops",
101+
help=(
102+
"Useful argument to take BUCK targets that registers prim ops and error out if we have more than 1."
103+
),
104+
required=False,
105+
)
98106
options = parser.parse_args(argv)
99107

108+
# Error out if we have more than one targets registering prim ops. The targets are likely
109+
if options.DEBUG_ONLY_check_prim_ops and len(options.DEBUG_ONLY_check_prim_ops) > 1:
110+
assert (
111+
options.DEBUG_ONLY_check_prim_ops[0] == "@"
112+
), "DEBUG_ONLY_check_prim_ops is not a valid file path, or it doesn't start with '@'. This is likely a BUCK issue."
113+
114+
prim_ops_targets_file = options.DEBUG_ONLY_check_prim_ops[1:]
115+
with open(prim_ops_targets_file, "r") as file:
116+
prim_ops_targets = file.read().split()
117+
if len(prim_ops_targets) > 1:
118+
# Yellow bold: \033[33;1m
119+
# Red bold: \033[31;1m
120+
# Green bold: \033[32;1m
121+
error = "It seems this target is depending on more than 1 `prim_ops_registry` targets: " \
122+
+ f'\033[33;1m\n{", ".join(prim_ops_targets)}\033[0m. \nThis will likely cause errors such as: ' \
123+
+ "\n \033[31;1mRe-registering aten::sym_size.int...\033[0m" \
124+
+ "\nTo find out the dependency chain, run the following command: " \
125+
+ f'\n \033[32;1mbuck2 cquery <mode> "allpaths(<target>, {prim_ops_targets[0]})"\033[0m'
126+
raise Exception(error)
100127
# Check if the build has any dependency on any selective build target. If we have a target, BUCK shold give us either:
101128
# 1. a yaml file containing selected ops (could be empty), or
102129
# 2. a non-empty list of yaml files in the `model_file_list_path` or
@@ -153,14 +180,17 @@ def main(argv: List[Any]) -> None:
153180
debug_info_2 = ",".join(
154181
model_dict["operators"][op_name]["debug_info"]
155182
)
156-
error = f"Operator {op_name} is used in 2 models: {debug_info_1} and {debug_info_2}"
183+
# Yellow bold: \033[33;1m
184+
# Red bold: \033[31;1m
185+
# Green bold: \033[32;1m
186+
error = f"\033[31;1mOperator {op_name} is used in 2 models: \033[33;1m{debug_info_1} and {debug_info_2}\033[0m"
157187
if "//" not in debug_info_1 and "//" not in debug_info_2:
158188
error += "\nWe can't determine what BUCK targets these model files belong to."
159189
tail = "."
160190
else:
161191
error += "\nPlease run the following commands to find out where is the BUCK target being added as a dependency to your target:\n"
162-
error += f'\n buck2 cquery <mode> "allpaths(<target>, {debug_info_1})"'
163-
error += f'\n buck2 cquery <mode> "allpaths(<target>, {debug_info_2})"'
192+
error += f'\n \033[32;1mbuck2 cquery <mode> "allpaths(<target>, {debug_info_1})"\033[0m'
193+
error += f'\n \033[32;1mbuck2 cquery <mode> "allpaths(<target>, {debug_info_2})"\033[0m'
164194
tail = "as well as results from BUCK commands listed above."
165195

166196
error += (

shim/xplat/executorch/codegen/codegen.bzl

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -689,9 +689,10 @@ def executorch_ops_check(
689689
name = name,
690690
macros_only = False,
691691
cmd = ("$(exe fbsource//xplat/executorch/codegen/tools:gen_all_oplist) " +
692-
"--model_file_list_path $(@query_outputs \"filter('.*_et_oplist', deps(set({deps})))\") " +
692+
"--model_file_list_path $(@query_targets \"filter('.*_et_oplist', deps(set({deps})))\") " +
693693
"--allow_include_all_overloads " +
694694
"--check_ops_not_overlapping " +
695+
"--DEBUG_ONLY_check_prim_ops $(@query_targets \"filter('prim_ops_registry(?:_static|_aten)?$', deps(set({deps})))\") " +
695696
"--output_dir $OUT ").format(deps = " ".join(["\'{}\'".format(d) for d in deps])),
696697
define_static_target = False,
697698
platforms = kwargs.pop("platforms", get_default_executorch_platforms()),

0 commit comments

Comments
 (0)