Buck build - fix use_tiktoken config

Summary:
Make it work

bypass-github-export-checks

Reviewed By: larryliu0820

Differential Revision: D56287998

fbshipit-source-id: 02b92c8110f7ea72055edd4c194858cc71b49093

Author: digantdesai

examples/models/llama2/runner/runner.cpp

@@ -10,10 +10,11 @@
 // The module takes in a string as input and emits a string as output.
 
 #include <executorch/examples/models/llama2/runner/runner.h>
-#include <executorch/examples/models/llama2/tokenizer/bpe_tokenizer.h>
 #if defined(ET_USE_TIKTOKEN)
 #include <executorch/examples/models/llama2/tokenizer/tiktoken.h>
-#endif
+#else /* BPE */
+#include <executorch/examples/models/llama2/tokenizer/bpe_tokenizer.h>
+#endif /* ET_USE_TIKTOKEN*/
 #include <executorch/extension/evalue_util/print_evalue.h>
 #include <executorch/extension/runner_util/managed_tensor.h>
 

examples/models/llama2/runner/targets.bzl

@@ -8,6 +8,9 @@ def _get_operator_lib(aten = False):
     else:
         return ["//executorch/configurations:optimized_native_cpu_ops", "//executorch/examples/models/llama2/custom_ops:custom_ops"]
 
+def use_tiktoken():
+    return native.read_config("llama", "use_tiktoken", "0") == "1"
+
 def define_common_targets():
     for aten in (True, False):
         aten_suffix = "_aten" if aten else ""
@@ -24,6 +27,7 @@ def define_common_targets():
             preprocessor_flags = [
                 "-DUSE_ATEN_LIB",
             ] if aten else [],
+            exported_preprocessor_flags = ["-DET_USE_TIKTOKEN"] if use_tiktoken() else [],
             visibility = [
                 "@EXECUTORCH_CLIENTS",
             ],
@@ -38,7 +42,7 @@ def define_common_targets():
                 "//executorch/runtime/core/exec_aten/util:tensor_util" + aten_suffix,
             ] + ([
                 "//executorch/examples/models/llama2/tokenizer:tiktoken",
-            ] if native.read_config("llama", "use_tiktoken", "0") == "1" else [
+            ] if use_tiktoken() else [
                 "//executorch/examples/models/llama2/tokenizer:bpe_tokenizer",
             ]) + (_get_operator_lib(aten)) + ([
                 # Vulkan API currently cannot build on some platforms (e.g. Apple, FBCODE)