Revert "Use pytorch-labs/tokenizers and remove tokenizer/ (#1401)" (#1414)

This reverts commit fff956c.

Author: Jack-Khuu

.github/workflows/pull.yml

@@ -942,7 +942,7 @@ jobs:
           path: |
             ./et-build
             ./torchchat/utils/scripts
-          key: et-build-${{runner.os}}-${{runner.arch}}-${{env.et-git-hash}}-${{ hashFiles('**/install_et.sh', '**/build_native.sh') }}
+          key: et-build-${{runner.os}}-${{runner.arch}}-${{env.et-git-hash}}-${{ hashFiles('**/install_et.sh') }}
       - if: ${{ steps.install-et.outputs.cache-hit != 'true' }}
         continue-on-error: true
         run: |
@@ -1053,7 +1053,7 @@ jobs:
 
           # Pull submodules (re2, abseil) for Tiktoken
           git submodule sync
-          git submodule update --init --recursive
+          git submodule update --init
           ./runner/build_android.sh
           echo "Tests complete."
 

.gitmodules

@@ -1,3 +1,9 @@
-[submodule "runner/third-party/tokenizers"]
-	path = runner/third-party/tokenizers
-	url = https://github.com/pytorch-labs/tokenizers
+[submodule "tokenizer/third-party/abseil-cpp"]
+	path = tokenizer/third-party/abseil-cpp
+	url = https://github.com/abseil/abseil-cpp.git
+[submodule "tokenizer/third-party/re2"]
+	path = tokenizer/third-party/re2
+	url = https://github.com/google/re2.git
+[submodule "tokenizer/third-party/sentencepiece"]
+	path = tokenizer/third-party/sentencepiece
+	url = https://github.com/google/sentencepiece.git

CMakeLists.txt

@@ -7,21 +7,18 @@ ELSE()
 ENDIF()
 
 project(Torchchat)
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-attributes")
 
 # include tokenizer
-add_subdirectory(runner/third-party/tokenizers)
+add_subdirectory(tokenizer)
 
 # include et_run executable
 include(runner/et.cmake)
 if(TARGET et_run)
-    target_link_libraries(et_run PUBLIC tokenizers microkernels-prod)
-    target_include_directories(et_run PUBLIC runner/third-party/tokenizers/include)
+    target_link_libraries(et_run PUBLIC tokenizer microkernels-prod)
 endif()
 
 # include aoti_run executable
 include(runner/aoti.cmake)
 if(TARGET aoti_run)
-    target_link_libraries(aoti_run tokenizers)
-    target_include_directories(aoti_run PUBLIC runner/third-party/tokenizers/include)
+    target_link_libraries(aoti_run tokenizer)
 endif()