Skip to content

Commit b033b2f

Browse files
larryliu0820facebook-github-bot
larryliu0820
authored and
facebook-github-bot
committed
Add llava runner test binary and build script (#4750)
Summary: Pull Request resolved: #4750 Pull Request resolved: #4667 Add a `main.cpp` and CMakeLists.txt for llava runner. This runner takes in an image in the format of `.pt` (a serialized pytorch model) along with text prompt. It will generate text tokens in a way similar to llama runner. Run `build.sh` to build the runner. To serialize the image into a `.pt` file, run the following script: ```python import torch from torch import nn copy = torch.tensor(resized) m = nn.Module() par = nn.Parameter(copy, requires_grad=False) m.register_parameter("0",par) tensors = torch.jit.script(m) tensors.save("image.pt") ``` To run the runner, use the following command: ``` cmake-out/examples/models/llava/llava_main \ --tokenizer_path tokenizer.bin \ --model_path llava_kv_768.pte \ --prompt "\nWhat are the things I should be cautious about when I visit here?" \ --image_path image.pt \ --temperature 0 ``` imported-using-ghimport Test Plan: Imported from OSS Reviewed By: kirklandsign Differential Revision: D61146432 Pulled By: larryliu0820
1 parent 5c9a00a commit b033b2f

File tree

13 files changed

+672
-41
lines changed

13 files changed

+672
-41
lines changed

‎.ci/scripts/test_llava.sh

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
set -exu
9+
# shellcheck source=/dev/null
10+
11+
if [[ -z "${PYTHON_EXECUTABLE:-}" ]]; then
12+
PYTHON_EXECUTABLE=python3
13+
fi
14+
15+
cmake_install_executorch_libraries() {
16+
cmake \
17+
-DCMAKE_INSTALL_PREFIX=cmake-out \
18+
-DCMAKE_BUILD_TYPE=Debug \
19+
-DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
20+
-DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
21+
-DEXECUTORCH_BUILD_KERNELS_CUSTOM=ON \
22+
-DEXECUTORCH_BUILD_KERNELS_OPTIMIZED=ON \
23+
-DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON \
24+
-DEXECUTORCH_BUILD_XNNPACK=ON \
25+
-DEXECUTORCH_DO_NOT_USE_CXX11_ABI=ON \
26+
-Bcmake-out .
27+
28+
29+
cmake --build cmake-out -j9 --target install --config Debug
30+
}
31+
32+
cmake_build_llava_runner() {
33+
dir=examples/models/llava
34+
python_lib=$($PYTHON_EXECUTABLE -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')
35+
36+
cmake \
37+
-DCMAKE_INSTALL_PREFIX=cmake-out \
38+
-DCMAKE_BUILD_TYPE=Debug \
39+
-DEXECUTORCH_BUILD_KERNELS_CUSTOM=ON \
40+
-DEXECUTORCH_BUILD_KERNELS_OPTIMIZED=ON \
41+
-DEXECUTORCH_BUILD_XNNPACK=ON \
42+
-DCMAKE_PREFIX_PATH="$python_lib" \
43+
-Bcmake-out/${dir} \
44+
${dir}
45+
46+
47+
cmake --build cmake-out/${dir} -j9 --config Debug
48+
}
49+
50+
# only export the one without custom op for now since it's
51+
export_llava() {
52+
echo "Starting to export Llava. This will take about 6 mins"
53+
$PYTHON_EXECUTABLE -m executorch.examples.models.llava.export_llava --pte-name llava.pte --with-artifacts
54+
}
55+
56+
run_and_verify() {
57+
NOW=$(date +"%H:%M:%S")
58+
echo "Starting to run llava runner at ${NOW}"
59+
if [[ ! -f "llava.pte" ]]; then
60+
echo "Export failed. Abort"
61+
exit 1
62+
fi
63+
if [[ ! -f "image.pt" ]]; then
64+
echo "image.pt is missing."
65+
exit 1
66+
fi
67+
if [[ ! -f "tokenizer.bin" ]]; then
68+
echo "tokenizer.bin is missing."
69+
exit 1
70+
fi
71+
RUNTIME_ARGS="--model_path=llava.pte \
72+
--tokenizer_path=tokenizer.bin \
73+
--image_path=image.pt \
74+
--prompt=\"What are the things I should be cautious about when I visit here? ASSISTANT: \" \
75+
--temperature=0 \
76+
--seq_len=650"
77+
cmake-out/examples/models/llava/llava_main ${RUNTIME_ARGS} > result.txt
78+
# verify result.txt
79+
RESULT=$(cat result.txt)
80+
# set the expected prefix to be the same as prompt because there's a bug in sdpa_with_kv_cache that causes <unk> tokens.
81+
EXPECTED_PREFIX="What are the things I should be cautious about when I visit here? ASSISTANT: "
82+
if [[ "${RESULT}" == "${EXPECTED_PREFIX}"* ]]; then
83+
echo "Expected result prefix: ${EXPECTED_PREFIX}"
84+
echo "Actual result: ${RESULT}"
85+
echo "Success"
86+
exit 0
87+
else
88+
echo "Expected result prefix: ${EXPECTED_PREFIX}"
89+
echo "Actual result: ${RESULT}"
90+
echo "Failure; results not the same"
91+
exit 1
92+
fi
93+
}
94+
95+
cmake_install_executorch_libraries
96+
cmake_build_llava_runner
97+
# export_llava
98+
run_and_verify

‎.github/workflows/pull.yml

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -187,7 +187,7 @@ jobs:
187187
# Test selective build
188188
PYTHON_EXECUTABLE=python bash examples/selective_build/test_selective_build.sh "${BUILD_TOOL}"
189189
190-
test-export-llava-linux:
190+
test-llava-runner-linux:
191191
name: test-export-llava-linux
192192
uses: pytorch/test-infra/.github/workflows/linux_job.yml@main
193193
strategy:
@@ -215,6 +215,9 @@ jobs:
215215
# run python unittest
216216
python -m unittest examples.models.llava.test.test_llava
217217
218+
# run e2e (export, tokenizer and runner)
219+
PYTHON_EXECUTABLE=python bash .ci/scripts/test_llava.sh
220+
218221
test-quantized-aot-lib-linux:
219222
name: test-quantized-aot-lib-linux
220223
uses: pytorch/test-infra/.github/workflows/linux_job.yml@main

‎CMakeLists.txt

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
# @lint-ignore-every LICENSELINT
12
# Copyright (c) Meta Platforms, Inc. and affiliates.
23
# All rights reserved.
34
#
@@ -130,6 +131,12 @@ if(EXECUTORCH_ENABLE_EVENT_TRACER)
130131
add_definitions(-DET_EVENT_TRACER_ENABLED)
131132
endif()
132133

134+
option(EXECUTORCH_DO_NOT_USE_CXX11_ABI "Define _GLIBCXX_USE_CXX11_ABI=0 if ON"
135+
OFF
136+
)
137+
if(EXECUTORCH_DO_NOT_USE_CXX11_ABI)
138+
add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0)
139+
endif()
133140
# -ffunction-sections -fdata-sections: breaks function and data into sections so
134141
# they can be properly gc'd. -s: strip symbol. -fno-exceptions -fno-rtti:
135142
# disables exceptions and runtime type.

‎examples/models/llava/CMakeLists.txt

Lines changed: 219 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,219 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
#
8+
# Simple CMake build system for llava runner.
9+
#
10+
# ### Editing this file ###
11+
#
12+
# This file should be formatted with
13+
# ~~~
14+
# cmake-format -i CMakeLists.txt
15+
# ~~~
16+
# It should also be cmake-lint clean.
17+
#
18+
cmake_minimum_required(VERSION 3.19)
19+
project(llava)
20+
21+
# Duplicating options as root CMakeLists.txt
22+
option(EXECUTORCH_BUILD_KERNELS_OPTIMIZED "Build the optimized kernels" OFF)
23+
24+
25+
include(CMakeDependentOption)
26+
#
27+
# pthreadpool: build pthreadpool library. Disable on unsupported platforms
28+
#
29+
cmake_dependent_option(
30+
EXECUTORCH_BUILD_PTHREADPOOL "Build pthreadpool library." ON
31+
"NOT EXECUTORCH_BUILD_ARM_BAREMETAL" OFF
32+
)
33+
#
34+
# cpuinfo: build cpuinfo library. Disable on unsupported platforms
35+
#
36+
cmake_dependent_option(
37+
EXECUTORCH_BUILD_CPUINFO "Build cpuinfo library." ON
38+
"NOT EXECUTORCH_BUILD_ARM_BAREMETAL" OFF
39+
)
40+
41+
if(NOT PYTHON_EXECUTABLE)
42+
set(PYTHON_EXECUTABLE python3)
43+
endif()
44+
45+
set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../../..)
46+
set(TORCH_ROOT ${EXECUTORCH_ROOT}/third-party/pytorch)
47+
48+
include(${EXECUTORCH_ROOT}/build/Utils.cmake)
49+
50+
if(NOT PYTHON_EXECUTABLE)
51+
resolve_python_executable()
52+
endif()
53+
54+
if(NOT CMAKE_CXX_STANDARD)
55+
set(CMAKE_CXX_STANDARD 17)
56+
# Can't set to 11 due to executor_runner.cpp make_unique
57+
endif()
58+
59+
if(CMAKE_TOOLCHAIN_FILE MATCHES ".*(iOS|ios\.toolchain)\.cmake$")
60+
set(CMAKE_TOOLCHAIN_IOS ON)
61+
else()
62+
set(CMAKE_TOOLCHAIN_IOS OFF)
63+
endif()
64+
65+
set(_common_compile_options -Wno-deprecated-declarations -fPIC)
66+
67+
# Let files say "include <executorch/path/to/header.h>".
68+
set(_common_include_directories ${EXECUTORCH_ROOT}/..)
69+
70+
# For some reason android build is not able to find where gflags is and hence
71+
# cannot find corresponding .cmake file
72+
set(gflags_DIR ${CMAKE_CURRENT_BINARY_DIR}/../../../third-party/gflags)
73+
find_package(gflags REQUIRED)
74+
75+
find_package(Torch CONFIG REQUIRED)
76+
add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0)
77+
78+
#
79+
# llava_main: test binary to run llava, with tokenizer and sampler integrated
80+
#
81+
82+
# find `executorch` libraries Same as for gflags
83+
set(executorch_DIR ${CMAKE_CURRENT_BINARY_DIR}/../../../lib/cmake/ExecuTorch)
84+
find_package(executorch CONFIG REQUIRED)
85+
if(CMAKE_TOOLCHAIN_IOS OR ANDROID)
86+
target_link_options_shared_lib(executorch)
87+
endif()
88+
89+
# custom ops library
90+
if(EXECUTORCH_BUILD_KERNELS_CUSTOM)
91+
add_subdirectory(
92+
${EXECUTORCH_ROOT}/extension/llm/custom_ops
93+
${CMAKE_CURRENT_BINARY_DIR}/../../../extension/llm/custom_ops
94+
)
95+
endif()
96+
97+
# llava_runner library
98+
add_subdirectory(runner)
99+
100+
set(link_libraries gflags torch)
101+
set(_srcs main.cpp)
102+
103+
if(EXECUTORCH_BUILD_KERNELS_OPTIMIZED)
104+
list(
105+
APPEND
106+
link_libraries
107+
optimized_native_cpu_ops_lib
108+
optimized_kernels
109+
portable_kernels
110+
cpublas
111+
eigen_blas
112+
)
113+
target_link_options_shared_lib(optimized_native_cpu_ops_lib)
114+
else()
115+
list(APPEND link_libraries portable_ops_lib portable_kernels)
116+
target_link_options_shared_lib(portable_ops_lib)
117+
endif()
118+
119+
# quantized_ops_lib: Register quantized op kernels into the runtime
120+
target_link_options_shared_lib(quantized_ops_lib)
121+
list(APPEND link_libraries quantized_kernels quantized_ops_lib)
122+
123+
if(EXECUTORCH_BUILD_KERNELS_CUSTOM)
124+
target_link_options_shared_lib(custom_ops)
125+
list(APPEND link_libraries custom_ops)
126+
endif()
127+
128+
set(XNNPACK_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../../../backends/xnnpack)
129+
# Extra compile option and include dir for pthreadpool
130+
if(EXECUTORCH_BUILD_PTHREADPOOL)
131+
list(APPEND _common_compile_options -DET_USE_THREADPOOL)
132+
list(APPEND link_libraries pthreadpool)
133+
# These 2 source files are included in xnnpack_backend
134+
if(NOT TARGET xnnpack_backend)
135+
list(APPEND _srcs ${XNNPACK_ROOT}/threadpool/threadpool.cpp
136+
${XNNPACK_ROOT}/threadpool/threadpool_guard.cpp
137+
)
138+
endif()
139+
list(APPEND _common_include_directories
140+
${XNNPACK_ROOT}/third-party/pthreadpool/include
141+
)
142+
endif()
143+
144+
# Extra sources for cpuinfo
145+
if(EXECUTORCH_BUILD_CPUINFO)
146+
list(APPEND link_libraries cpuinfo)
147+
list(APPEND _srcs ${XNNPACK_ROOT}/threadpool/cpuinfo_utils.cpp)
148+
list(APPEND _common_include_directories
149+
${XNNPACK_ROOT}/third-party/cpuinfo/include
150+
)
151+
endif()
152+
153+
# XNNPACK
154+
if(TARGET xnnpack_backend)
155+
set(xnnpack_backend_libs xnnpack_backend XNNPACK)
156+
list(APPEND link_libraries ${xnnpack_backend_libs})
157+
target_link_options_shared_lib(xnnpack_backend)
158+
endif()
159+
160+
# Vulkan backend
161+
if(TARGET vulkan_backend)
162+
list(APPEND link_libraries vulkan_backend)
163+
target_link_options_shared_lib(vulkan_backend)
164+
endif()
165+
166+
# Qnn backend
167+
if(TARGET qnn_executorch_backend)
168+
list(APPEND link_libraries qnn_executorch_backend)
169+
target_link_options_shared_lib(qnn_executorch_backend)
170+
endif()
171+
172+
# MPS backend
173+
if(TARGET mpsdelegate)
174+
list(
175+
APPEND
176+
link_libraries
177+
mpsdelegate
178+
"-framework Foundation"
179+
"-weak_framework MetalPerformanceShaders"
180+
"-weak_framework MetalPerformanceShadersGraph"
181+
"-weak_framework Metal"
182+
)
183+
target_link_options_shared_lib(mpsdelegate)
184+
endif()
185+
186+
if(TARGET coremldelegate)
187+
find_library(SQLITE_LIBRARY sqlite3)
188+
list(
189+
APPEND
190+
link_libraries
191+
coremldelegate
192+
sqlite3
193+
"-framework Foundation"
194+
"-framework CoreML"
195+
"-framework Accelerate"
196+
)
197+
target_link_options_shared_lib(coremldelegate)
198+
endif()
199+
200+
# This one is needed for cpuinfo where it uses android specific log lib
201+
if(ANDROID)
202+
list(APPEND link_libraries log)
203+
endif()
204+
205+
add_executable(llava_main ${_srcs})
206+
if(CMAKE_BUILD_TYPE STREQUAL "Release")
207+
target_link_options(llava_main PRIVATE "LINKER:--gc-sections,-s")
208+
endif()
209+
210+
target_include_directories(llava_main PUBLIC ${_common_include_directories})
211+
target_link_libraries(llava_main PUBLIC llava_runner ${link_libraries})
212+
target_compile_options(llava_main PUBLIC ${_common_compile_options})
213+
214+
if(APPLE)
215+
target_link_options_shared_lib(executorch)
216+
endif()
217+
218+
# Print all summary
219+
executorch_print_configuration_summary()

0 commit comments

Comments
 (0)