Fix build for tokenizer tool script (#36)

Author: jackzhxng

CMakeLists.txt

@@ -86,5 +86,5 @@ endif()
 
 # Build tools
 if(TOKENIZERS_BUILD_TOOLS)
-  add_subdirectory(tools/tokenize_tool)
+  add_subdirectory(examples/tokenize_tool)
 endif()

examples/tokenize_tool/CMakeLists.txt

@@ -8,3 +8,6 @@ file(GLOB source_files ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp)
 get_filename_component(tool_name ${CMAKE_CURRENT_SOURCE_DIR} NAME)
 add_executable(${tool_name} ${source_files})
 target_link_libraries(${tool_name} PRIVATE tokenizers)
+target_include_directories(${tool_name} PRIVATE
+    ${CMAKE_SOURCE_DIR}/include/pytorch/tokenizers
+)