Cmake

Author: jackzhxng

examples/models/llama/runner/CMakeLists.txt

@@ -47,10 +47,6 @@ list(
 )
 list(APPEND _llama_runner__srcs
      ${CMAKE_CURRENT_SOURCE_DIR}/../tokenizer/llama_tiktoken.cpp
-   )
-list(
-  APPEND _llama_runner__srcs
-  ${CMAKE_CURRENT_SOURCE_DIR}/../../../../extension/llm/tokenizer/hf_tokenizer.cpp
 )
 
 if(CMAKE_TOOLCHAIN_IOS
@@ -77,10 +73,19 @@ add_subdirectory(
   ${EXECUTORCH_ROOT}/extension/llm/tokenizers/third-party/re2
   ${CMAKE_CURRENT_BINARY_DIR}/re2
 )
+add_subdirectory(
+  ${EXECUTORCH_ROOT}/extension/llm/tokenizers/third-party/json
+  ${CMAKE_CURRENT_BINARY_DIR}/json
+)
+target_include_directories(llama_runner
+  PRIVATE ${CMAKE_INSTALL_PREFIX}/include
+  ${EXECUTORCH_ROOT}/extension/llm/tokenizers/third-party/llama.cpp-unicode/include
+  ${EXECUTORCH_ROOT}/extension/llm/tokenizers/third-party/llama.cpp-unicode/src
+)
 set(CMAKE_POSITION_INDEPENDENT_CODE ${_pic_flag})
 
 set(llama_runner_deps executorch extension_data_loader extension_module
-                      extension_tensor re2::re2
+                      extension_tensor re2::re2 nlohmann_json::nlohmann_json
 )
 
 target_link_libraries(llama_runner PUBLIC ${llama_runner_deps})

examples/models/llama/runner/runner.cpp

@@ -82,8 +82,10 @@ Error Runner::load() {
   tokenizer_ = nullptr;
   // Check if tokenizer_path_ ends with ".json".
   if (tokenizer_path_.size() >= 5 &&
+      
       tokenizer_path_.compare(tokenizer_path_.size() - 5, 5, ".json") == 0) {
     tokenizer_ = std::make_unique<tokenizers::HFTokenizer>();
+    ET_LOG(Info, "Loading json tokenizer");
     tokenizer_->load(tokenizer_path_);
     ET_LOG(
         Info, "Loaded tokenizer %s as HF tokenizer", tokenizer_path_.c_str());

examples/models/llama/runner/targets.bzl

@@ -49,7 +49,7 @@ def define_common_targets():
                 "//executorch/runtime/core/exec_aten/util:tensor_util" + aten_suffix,
                 "//executorch/examples/models/llama/tokenizer:tiktoken",
                 "//pytorch/tokenizers:llama2c_tokenizer",
-		"//pytorch/tokenizers:hf_tokenizer",
+                "//pytorch/tokenizers:hf_tokenizer",
             ] + (_get_operator_lib(aten)) + ([
                 # Vulkan API currently cannot build on some platforms (e.g. Apple, FBCODE)
                 # Therefore enable it explicitly for now to avoid failing tests

extension/llm/runner/CMakeLists.txt

@@ -49,6 +49,13 @@ set(runner_deps executorch extension_data_loader extension_module
 
 target_link_libraries(extension_llm_runner PUBLIC ${runner_deps})
 
+target_include_directories(
+  extension_llm_runner
+  PUBLIC
+    ${EXECUTORCH_ROOT}/extension/llm/tokenizers/third-party/llama.cpp-unicode/include
+    ${EXECUTORCH_ROOT}/extension/llm/tokenizers/third-party/llama.cpp-unicode/src
+)
+
 target_include_directories(
   extension_llm_runner INTERFACE ${_common_include_directories}
                                  ${EXECUTORCH_ROOT}/extension/llm/tokenizers/include