Skip to content

Commit 5a643b3

Browse files
jackzhxngjackzhxng
authored
Fix llama app build (#10583)
- Pull in recent tokenizer changes - Fix error where `llama_tiktoken.h/cpp` files couldn't be found
1 parent 9a18669 commit 5a643b3

File tree

1 file changed

+63
-3
lines changed

1 file changed

+63
-3
lines changed

‎examples/demo-apps/apple_ios/LLaMA/LLaMA.xcodeproj/project.pbxproj

Lines changed: 63 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,21 @@
4747
03D03DAB2C7823830088D6A7 /* text_decoder_runner.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 03D03DA92C7823830088D6A7 /* text_decoder_runner.cpp */; };
4848
03D03DAC2C7823830088D6A7 /* text_decoder_runner.h in Headers */ = {isa = PBXBuildFile; fileRef = 03D03DAA2C7823830088D6A7 /* text_decoder_runner.h */; };
4949
26A6A4282C8A3769005A761E /* ImagePicker.swift in Sources */ = {isa = PBXBuildFile; fileRef = 26A6A4272C8A3769005A761E /* ImagePicker.swift */; };
50+
306A713D2DC1DC0F00936B1F /* token_decoder.h in Headers */ = {isa = PBXBuildFile; fileRef = 306A713C2DC1DC0F00936B1F /* token_decoder.h */; };
51+
306A713E2DC1DC0F00936B1F /* regex.h in Headers */ = {isa = PBXBuildFile; fileRef = 306A71392DC1DC0F00936B1F /* regex.h */; };
52+
306A713F2DC1DC0F00936B1F /* string_integer_map.h in Headers */ = {isa = PBXBuildFile; fileRef = 306A713B2DC1DC0F00936B1F /* string_integer_map.h */; };
53+
306A71402DC1DC0F00936B1F /* re2_regex.h in Headers */ = {isa = PBXBuildFile; fileRef = 306A71382DC1DC0F00936B1F /* re2_regex.h */; };
54+
306A71412DC1DC0F00936B1F /* hf_tokenizer.h in Headers */ = {isa = PBXBuildFile; fileRef = 306A71352DC1DC0F00936B1F /* hf_tokenizer.h */; };
55+
306A71422DC1DC0F00936B1F /* pre_tokenizer.h in Headers */ = {isa = PBXBuildFile; fileRef = 306A71372DC1DC0F00936B1F /* pre_tokenizer.h */; };
56+
306A71432DC1DC0F00936B1F /* pcre2_regex.h in Headers */ = {isa = PBXBuildFile; fileRef = 306A71362DC1DC0F00936B1F /* pcre2_regex.h */; };
57+
306A71442DC1DC0F00936B1F /* std_regex.h in Headers */ = {isa = PBXBuildFile; fileRef = 306A713A2DC1DC0F00936B1F /* std_regex.h */; };
58+
306A714C2DC1DC3D00936B1F /* std_regex.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 306A714A2DC1DC3D00936B1F /* std_regex.cpp */; };
59+
306A714D2DC1DC3D00936B1F /* re2_regex.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 306A71482DC1DC3D00936B1F /* re2_regex.cpp */; };
60+
306A714E2DC1DC3D00936B1F /* pcre2_regex.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 306A71462DC1DC3D00936B1F /* pcre2_regex.cpp */; };
61+
306A714F2DC1DC3D00936B1F /* hf_tokenizer.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 306A71452DC1DC3D00936B1F /* hf_tokenizer.cpp */; };
62+
306A71502DC1DC3D00936B1F /* regex.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 306A71492DC1DC3D00936B1F /* regex.cpp */; };
63+
306A71512DC1DC3D00936B1F /* pre_tokenizer.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 306A71472DC1DC3D00936B1F /* pre_tokenizer.cpp */; };
64+
306A71522DC1DC3D00936B1F /* token_decoder.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 306A714B2DC1DC3D00936B1F /* token_decoder.cpp */; };
5065
F292B0752D88B0C200BE6839 /* tiktoken.cpp in Sources */ = {isa = PBXBuildFile; fileRef = F292B06F2D88B0C200BE6839 /* tiktoken.cpp */; };
5166
F292B0762D88B0C200BE6839 /* llama2c_tokenizer.cpp in Sources */ = {isa = PBXBuildFile; fileRef = F292B06C2D88B0C200BE6839 /* llama2c_tokenizer.cpp */; };
5267
F292B0772D88B0C200BE6839 /* bpe_tokenizer_base.cpp in Sources */ = {isa = PBXBuildFile; fileRef = F292B06A2D88B0C200BE6839 /* bpe_tokenizer_base.cpp */; };
@@ -117,6 +132,21 @@
117132
03D03DA92C7823830088D6A7 /* text_decoder_runner.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = text_decoder_runner.cpp; sourceTree = "<group>"; };
118133
03D03DAA2C7823830088D6A7 /* text_decoder_runner.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = text_decoder_runner.h; sourceTree = "<group>"; };
119134
26A6A4272C8A3769005A761E /* ImagePicker.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ImagePicker.swift; sourceTree = "<group>"; };
135+
306A71352DC1DC0F00936B1F /* hf_tokenizer.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = hf_tokenizer.h; sourceTree = "<group>"; };
136+
306A71362DC1DC0F00936B1F /* pcre2_regex.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = pcre2_regex.h; sourceTree = "<group>"; };
137+
306A71372DC1DC0F00936B1F /* pre_tokenizer.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = pre_tokenizer.h; sourceTree = "<group>"; };
138+
306A71382DC1DC0F00936B1F /* re2_regex.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = re2_regex.h; sourceTree = "<group>"; };
139+
306A71392DC1DC0F00936B1F /* regex.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = regex.h; sourceTree = "<group>"; };
140+
306A713A2DC1DC0F00936B1F /* std_regex.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = std_regex.h; sourceTree = "<group>"; };
141+
306A713B2DC1DC0F00936B1F /* string_integer_map.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = string_integer_map.h; sourceTree = "<group>"; };
142+
306A713C2DC1DC0F00936B1F /* token_decoder.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = token_decoder.h; sourceTree = "<group>"; };
143+
306A71452DC1DC3D00936B1F /* hf_tokenizer.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = hf_tokenizer.cpp; sourceTree = "<group>"; };
144+
306A71462DC1DC3D00936B1F /* pcre2_regex.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = pcre2_regex.cpp; sourceTree = "<group>"; };
145+
306A71472DC1DC3D00936B1F /* pre_tokenizer.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = pre_tokenizer.cpp; sourceTree = "<group>"; };
146+
306A71482DC1DC3D00936B1F /* re2_regex.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = re2_regex.cpp; sourceTree = "<group>"; };
147+
306A71492DC1DC3D00936B1F /* regex.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = regex.cpp; sourceTree = "<group>"; };
148+
306A714A2DC1DC3D00936B1F /* std_regex.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = std_regex.cpp; sourceTree = "<group>"; };
149+
306A714B2DC1DC3D00936B1F /* token_decoder.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = token_decoder.cpp; sourceTree = "<group>"; };
120150
F292B06A2D88B0C200BE6839 /* bpe_tokenizer_base.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = bpe_tokenizer_base.cpp; path = src/bpe_tokenizer_base.cpp; sourceTree = "<group>"; };
121151
F292B06C2D88B0C200BE6839 /* llama2c_tokenizer.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = llama2c_tokenizer.cpp; path = src/llama2c_tokenizer.cpp; sourceTree = "<group>"; };
122152
F292B06F2D88B0C200BE6839 /* tiktoken.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = tiktoken.cpp; path = src/tiktoken.cpp; sourceTree = "<group>"; };
@@ -128,8 +158,8 @@
128158
F292B07F2D88B0D200BE6839 /* result.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = result.h; sourceTree = "<group>"; };
129159
F292B0812D88B0D200BE6839 /* tiktoken.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = tiktoken.h; sourceTree = "<group>"; };
130160
F292B0832D88B0D200BE6839 /* tokenizer.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = tokenizer.h; sourceTree = "<group>"; };
131-
F292B0FF2D88B20C00BE6839 /* llama_tiktoken.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = llama_tiktoken.h; path = ../../../../../executorch/examples/models/llama/tokenizer/llama_tiktoken.h; sourceTree = "<group>"; };
132-
F292B1002D88B20C00BE6839 /* llama_tiktoken.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = llama_tiktoken.cpp; path = ../../../../../executorch/examples/models/llama/tokenizer/llama_tiktoken.cpp; sourceTree = "<group>"; };
161+
F292B0FF2D88B20C00BE6839 /* llama_tiktoken.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = llama_tiktoken.h; path = ../../../examples/models/llama/tokenizer/llama_tiktoken.h; sourceTree = "<group>"; };
162+
F292B1002D88B20C00BE6839 /* llama_tiktoken.cpp */ = {isa = PBXFileReference; explicitFileType = sourcecode.cpp.cpp; name = llama_tiktoken.cpp; path = ../../../examples/models/llama/tokenizer/llama_tiktoken.cpp; sourceTree = "<group>"; };
133163
/* End PBXFileReference section */
134164

135165
/* Begin PBXFrameworksBuildPhase section */
@@ -277,6 +307,13 @@
277307
03729F0E2BB203D700152F2E /* tokenizers */ = {
278308
isa = PBXGroup;
279309
children = (
310+
306A71452DC1DC3D00936B1F /* hf_tokenizer.cpp */,
311+
306A71462DC1DC3D00936B1F /* pcre2_regex.cpp */,
312+
306A71472DC1DC3D00936B1F /* pre_tokenizer.cpp */,
313+
306A71482DC1DC3D00936B1F /* re2_regex.cpp */,
314+
306A71492DC1DC3D00936B1F /* regex.cpp */,
315+
306A714A2DC1DC3D00936B1F /* std_regex.cpp */,
316+
306A714B2DC1DC3D00936B1F /* token_decoder.cpp */,
280317
F292B0FF2D88B20C00BE6839 /* llama_tiktoken.h */,
281318
F292B1002D88B20C00BE6839 /* llama_tiktoken.cpp */,
282319
F292B0862D88B0D200BE6839 /* include */,
@@ -301,6 +338,14 @@
301338
F292B0842D88B0D200BE6839 /* tokenizers */ = {
302339
isa = PBXGroup;
303340
children = (
341+
306A71352DC1DC0F00936B1F /* hf_tokenizer.h */,
342+
306A71362DC1DC0F00936B1F /* pcre2_regex.h */,
343+
306A71372DC1DC0F00936B1F /* pre_tokenizer.h */,
344+
306A71382DC1DC0F00936B1F /* re2_regex.h */,
345+
306A71392DC1DC0F00936B1F /* regex.h */,
346+
306A713A2DC1DC0F00936B1F /* std_regex.h */,
347+
306A713B2DC1DC0F00936B1F /* string_integer_map.h */,
348+
306A713C2DC1DC0F00936B1F /* token_decoder.h */,
304349
F292B0782D88B0D200BE6839 /* base64.h */,
305350
F292B0792D88B0D200BE6839 /* bpe_tokenizer_base.h */,
306351
F292B07A2D88B0D200BE6839 /* error.h */,
@@ -347,6 +392,14 @@
347392
F292B0912D88B0D200BE6839 /* tiktoken.h in Headers */,
348393
F292B0922D88B0D200BE6839 /* base64.h in Headers */,
349394
03D03DAC2C7823830088D6A7 /* text_decoder_runner.h in Headers */,
395+
306A713D2DC1DC0F00936B1F /* token_decoder.h in Headers */,
396+
306A713E2DC1DC0F00936B1F /* regex.h in Headers */,
397+
306A713F2DC1DC0F00936B1F /* string_integer_map.h in Headers */,
398+
306A71402DC1DC0F00936B1F /* re2_regex.h in Headers */,
399+
306A71412DC1DC0F00936B1F /* hf_tokenizer.h in Headers */,
400+
306A71422DC1DC0F00936B1F /* pre_tokenizer.h in Headers */,
401+
306A71432DC1DC0F00936B1F /* pcre2_regex.h in Headers */,
402+
306A71442DC1DC0F00936B1F /* std_regex.h in Headers */,
350403
03729F122BB2042B00152F2E /* sampler.h in Headers */,
351404
03729F0C2BB203B300152F2E /* util.h in Headers */,
352405
F292B1012D88B20C00BE6839 /* llama_tiktoken.h in Headers */,
@@ -491,7 +544,7 @@
491544
);
492545
runOnlyForDeploymentPostprocessing = 0;
493546
shellPath = /bin/sh;
494-
shellScript = "set -e\n\nif ! command -v cmake &> /dev/null\nthen\n echo \"Cmake not found, please install Cmake. \\n1. Download Cmake.app from https://cmake.org/download with version > 3.19. \\n2. Install it to Applications/ folder and run `sudo /Applications/CMake.app/Contents/bin/cmake-gui --install` to install CMake commandline tools.\"\n exit 1\nfi\n\nCMAKE_DIR=\"$TEMP_DIR/cmake\"\nrm -rf \"$CMAKE_DIR\"\n\nPLATFORM=\"SIMULATORARM64\"\nDEPLOYMENT_TARGET=\"17.0\"\n\nif [[ \"$PLATFORM_NAME\" == *\"iphoneos\"* ]]; then\n PLATFORM=\"OS64\"\nelif [[ \"$PLATFORM_NAME\" == *\"macos\"* ]]; then\n PLATFORM=\"MAC_ARM64\"\n DEPLOYMENT_TARGET=\"10.15\"\nfi\n\ncmake_build() {\n local src_dir=$1\n local target=$2\n shift 2\n local extra_args=(\"$@\")\n local build_dir=\"$CMAKE_DIR/build/$(basename \"$src_dir\")\"\n\n mkdir -p \"$build_dir\" && cd \"$build_dir\"\n\n if [[ \"$PLATFORM\" == \"MAC_ARM64\" ]]; then\n extra_args+=(-DCMAKE_INSTALL_BUNDLEDIR=\"${CMAKE_DIR}/bin\")\n extra_args+=(-DCMAKE_MACOSX_BUNDLE=OFF)\n fi\n cmake -G Xcode \\\n -DCMAKE_BUILD_TYPE=\"Release\" \\\n -DCMAKE_CXX_STANDARD=17 \\\n -DCMAKE_TOOLCHAIN_FILE=\"$SRCROOT/../../../../third-party/ios-cmake/ios.toolchain.cmake\" \\\n -DCMAKE_XCODE_ATTRIBUTE_CLANG_CXX_LANGUAGE_STANDARD=\"c++17\" \\\n -DCMAKE_XCODE_ATTRIBUTE_CLANG_CXX_LIBRARY=\"libc++\" \\\n -DPLATFORM=\"$PLATFORM\" \\\n -DDEPLOYMENT_TARGET=\"$DEPLOYMENT_TARGET\" \\\n -DCMAKE_INSTALL_PREFIX=\"$CMAKE_DIR\" \\\n \"${extra_args[@]}\" \\\n \"$src_dir\"\n cmake --build . --config \"Release\" --target \"$target\"\n if [[ \"$target\" == \"install\" ]]; then\n cmake --install . --prefix \"$CMAKE_DIR\"\n fi\n}\n\ncmake_build \"$SRCROOT/../../../../extension/llm/tokenizers/third-party/abseil-cpp\" \"install\" \\\n -DABSL_PROPAGATE_CXX_STD=ON\n\ncmake_build \"$SRCROOT/../../../../extension/llm/tokenizers/third-party/re2\" \"install\"\n\ncmake_build \"$SRCROOT/../../../../extension/llm/tokenizers/third-party/sentencepiece\" \"sentencepiece-static\" \\\n -DSPM_ENABLE_SHARED=OFF\n\necho \"$(find $CMAKE_DIR/lib -name \"*.a\" | sed -E 's|^.*/lib([^/]+)\\.a|-l\\1|g' | tr '\\n' ' ')\" > \"$CMAKE_DIR/linker_flags\"\n";
547+
shellScript = "set -e\n\nif ! command -v cmake &> /dev/null\nthen\n echo \"Cmake not found, please install Cmake. \\n1. Download Cmake.app from https://cmake.org/download with version > 3.19. \\n2. Install it to Applications/ folder and run `sudo /Applications/CMake.app/Contents/bin/cmake-gui --install` to install CMake commandline tools.\"\n exit 1\nfi\n\nCMAKE_DIR=\"$TEMP_DIR/cmake\"\nrm -rf \"$CMAKE_DIR\"\n\nPLATFORM=\"SIMULATORARM64\"\nDEPLOYMENT_TARGET=\"17.0\"\n\nif [[ \"$PLATFORM_NAME\" == *\"iphoneos\"* ]]; then\n PLATFORM=\"OS64\"\nelif [[ \"$PLATFORM_NAME\" == *\"macos\"* ]]; then\n PLATFORM=\"MAC_ARM64\"\n DEPLOYMENT_TARGET=\"10.15\"\nfi\n\ncmake_build() {\n local src_dir=$1\n local target=$2\n shift 2\n local extra_args=(\"$@\")\n local build_dir=\"$CMAKE_DIR/build/$(basename \"$src_dir\")\"\n\n mkdir -p \"$build_dir\" && cd \"$build_dir\"\n\n if [[ \"$PLATFORM\" == \"MAC_ARM64\" ]]; then\n extra_args+=(-DCMAKE_INSTALL_BUNDLEDIR=\"${CMAKE_DIR}/bin\")\n extra_args+=(-DCMAKE_MACOSX_BUNDLE=OFF)\n fi\n cmake -G Xcode \\\n -DCMAKE_BUILD_TYPE=\"Release\" \\\n -DCMAKE_CXX_STANDARD=17 \\\n -DCMAKE_TOOLCHAIN_FILE=\"$SRCROOT/../../../../third-party/ios-cmake/ios.toolchain.cmake\" \\\n -DCMAKE_XCODE_ATTRIBUTE_CLANG_CXX_LANGUAGE_STANDARD=\"c++17\" \\\n -DCMAKE_XCODE_ATTRIBUTE_CLANG_CXX_LIBRARY=\"libc++\" \\\n -DPLATFORM=\"$PLATFORM\" \\\n -DDEPLOYMENT_TARGET=\"$DEPLOYMENT_TARGET\" \\\n -DCMAKE_INSTALL_PREFIX=\"$CMAKE_DIR\" \\\n \"${extra_args[@]}\" \\\n \"$src_dir\"\n cmake --build . --config \"Release\" --target \"$target\"\n if [[ \"$target\" == \"install\" ]]; then\n cmake --install . --prefix \"$CMAKE_DIR\"\n fi\n}\n\ncmake_build \"$SRCROOT/../../../../extension/llm/tokenizers/third-party/abseil-cpp\" \"install\" \\\n -DABSL_PROPAGATE_CXX_STD=ON\n\ncmake_build \"$SRCROOT/../../../../extension/llm/tokenizers/third-party/re2\" \"install\"\n\ncmake_build \"$SRCROOT/../../../../extension/llm/tokenizers/third-party/pcre2\" \"install\" \\\n -DPCRE2_BUILD_PCRE2_8=ON \\\n -DPCRE2_BUILD_PCRE2_16=OFF \\\n -DPCRE2_BUILD_PCRE2_32=OFF \\\n -DPCRE2_BUILD_TESTS=OFF \\\n -DPCRE2_BUILD_PCRE2GREP=OFF \\\n -DPCRE2_BUILD_PCRE2TEST=OFF \\\n -DPCRE2_BUILD_PCRE2GPERF=OFF \\\n -DPCRE2_BUILD_DOCS=OFF \\\n -DPCRE2_BUILD_LIBPCRE2_PDB=OFF\n\ncmake_build \"$SRCROOT/../../../../extension/llm/tokenizers/third-party/sentencepiece\" \"sentencepiece-static\" \\\n -DSPM_ENABLE_SHARED=OFF\n \ncmake_build \"$SRCROOT/../../../../extension/llm/tokenizers/third-party/llama.cpp-unicode\" \"install\"\n \n# Include the single header for json.\nmkdir -p \"$CMAKE_DIR/include/nlohmann\"\ncp \"$SRCROOT/../../../../extension/llm/tokenizers/third-party/json/single_include/nlohmann/json.hpp\" \"$CMAKE_DIR/include/nlohmann/json.hpp\"\n\necho \"$(find $CMAKE_DIR/lib -name \"*.a\" | sed -E 's|^.*/lib([^/]+)\\.a|-l\\1|g' | tr '\\n' ' ')\" > \"$CMAKE_DIR/linker_flags\"\n";
495548
};
496549
/* End PBXShellScriptBuildPhase section */
497550

@@ -526,6 +579,13 @@
526579
F292B0772D88B0C200BE6839 /* bpe_tokenizer_base.cpp in Sources */,
527580
03729F0A2BB203B300152F2E /* runner.cpp in Sources */,
528581
03729F132BB2042B00152F2E /* sampler.cpp in Sources */,
582+
306A714C2DC1DC3D00936B1F /* std_regex.cpp in Sources */,
583+
306A714D2DC1DC3D00936B1F /* re2_regex.cpp in Sources */,
584+
306A714E2DC1DC3D00936B1F /* pcre2_regex.cpp in Sources */,
585+
306A714F2DC1DC3D00936B1F /* hf_tokenizer.cpp in Sources */,
586+
306A71502DC1DC3D00936B1F /* regex.cpp in Sources */,
587+
306A71512DC1DC3D00936B1F /* pre_tokenizer.cpp in Sources */,
588+
306A71522DC1DC3D00936B1F /* token_decoder.cpp in Sources */,
529589
03D03DA72C7823620088D6A7 /* text_prefiller.cpp in Sources */,
530590
);
531591
runOnlyForDeploymentPostprocessing = 0;

0 commit comments

Comments
 (0)