Update on "Fix test_llama_runner by hiding tiktoken"

Summary: We don't always want to build tiktoken dependencies (re2 and
abseil) so this PR only build it if the option is on.

Test Plan:

Reviewers:

Subscribers:

Tasks:

Tags:

Differential Revision: [D56178928](https://our.internmc.facebook.com/intern/diff/D56178928)

[ghstack-poisoned]

Author: larryliu0820

examples/models/llama2/runner/CMakeLists.txt

@@ -40,24 +40,25 @@ target_include_directories(extension_module
                            INTERFACE ${_common_include_directories})
 
 if(EXECUTORCH_USE_TIKTOKEN)
-  list(APPEND _llama_runner__srcs ${CMAKE_CURRENT_SOURCE_DIR}/../tokenizer/tiktoken.cpp)
+  list(APPEND _llama_runner__srcs
+       ${CMAKE_CURRENT_SOURCE_DIR}/../tokenizer/tiktoken.cpp)
   set(_preprocessor_flag -DET_USE_TIKTOKEN)
 endif()
 
-if(CMAKE_TOOLCHAIN_IOS OR ANDROID OR APPLE)
-  # Building a share library on iOS requires code signing
-  # On Android we see duplicated registration when using shared lib
+if(CMAKE_TOOLCHAIN_IOS
+   OR ANDROID
+   OR APPLE)
+  # Building a share library on iOS requires code signing On Android we see
+  # duplicated registration when using shared lib
   add_library(llama_runner STATIC ${_llama_runner__srcs})
 else()
   add_library(llama_runner SHARED ${_llama_runner__srcs})
 endif()
 
 set(llama_runner_deps executorch extension_module extension_data_loader)
 
-target_link_libraries(
-  llama_runner PUBLIC ${llama_runner_deps})
+target_link_libraries(llama_runner PUBLIC ${llama_runner_deps})
 
-target_include_directories(llama_runner
-                           INTERFACE ${_common_include_directories}
-                           ${EXECUTORCH_ROOT})
+target_include_directories(llama_runner INTERFACE ${_common_include_directories}
+                                                  ${EXECUTORCH_ROOT})
 target_compile_options(llama_runner PUBLIC ${_preprocessor_flag})

examples/models/llama2/runner/runner.h

@@ -85,7 +85,6 @@ class Runner {
   int32_t n_bos_;
   int32_t n_eos_;
   int32_t max_seq_len_;
-  bool use_tiktoken_;
   bool use_kv_cache_;
   bool use_sdpa_with_kv_cache_;
   bool append_eos_;