[llava] Use huggingface LLaVA instead of depending on third-party/LLaVa

Currently we depend on third-party/LLaVA for llava model definition.
This is hard to use because we have to pull LLaVA in as a git submodule
and install from there. It also breaks a lot of dependency assumptions.

This PR removes `third-party/LLaVA`, in favor of huggingface llava model
definition.

ghstack-source-id: fafdedd
Pull Request resolved: #4687

Author: larryliu0820

.gitmodules

@@ -28,9 +28,6 @@
 [submodule "backends/xnnpack/third-party/pthreadpool"]
 	path = backends/xnnpack/third-party/pthreadpool
 	url = https://github.com/Maratyszcza/pthreadpool.git
-[submodule "examples/third-party/LLaVA"]
-	path = examples/third-party/LLaVA
-	url = https://github.com/haotian-liu/LLaVA.git
 [submodule "examples/third-party/fbjni"]
 	path = examples/third-party/fbjni
 	url = https://github.com/facebookincubator/fbjni.git

examples/models/llava/export_llava.py

@@ -167,24 +167,7 @@ def export_token_embedding(llava, prompt):
     return token_embedding_ep
 
 
-def main():
-    parser = ArgumentParser()
-    parser.add_argument(
-        "--use-sdpa-with-kv-cache",
-        default=True,
-        action=BooleanOptionalAction,
-        help="Use sdpa_with_kv_cache custom op in LLava text model.",
-    )
-    parser.add_argument(
-        "--pte-name",
-        default="llava_combined_xnnpack.pte",
-        help="Name of the exported ExecuTorch program.",
-    )
-    args = parser.parse_args()
-    logging.info(
-        f"Exporting Llava model to ExecuTorch with sdpa_with_kv_cache: {args.use_sdpa_with_kv_cache}"
-    )
-    llava_model = LlavaModel(use_sdpa_with_kv_cache_op=args.use_sdpa_with_kv_cache)
+def export_all(llava_model: LlavaModel):
     llava = llava_model.get_eager_model()
 
     (
@@ -226,6 +209,29 @@ def main():
     )
 
     executorch_program = lowered_and_edge.to_executorch()
+    return executorch_program
+
+
+def main():
+    parser = ArgumentParser()
+    parser.add_argument(
+        "--use-sdpa-with-kv-cache",
+        default=True,
+        action=BooleanOptionalAction,
+        help="Use sdpa_with_kv_cache custom op in LLava text model.",
+    )
+    parser.add_argument(
+        "--pte-name",
+        default="llava_combined_xnnpack.pte",
+        help="Name of the exported ExecuTorch program.",
+    )
+    args = parser.parse_args()
+    logging.info(
+        f"Exporting Llava model to ExecuTorch with sdpa_with_kv_cache: {args.use_sdpa_with_kv_cache}"
+    )
+    llava_model = LlavaModel(use_sdpa_with_kv_cache_op=args.use_sdpa_with_kv_cache)
+
+    executorch_program = export_all(llava_model)
 
     with open(args.pte_name, "wb") as f:
         executorch_program.write_to_file(f)

examples/models/llava/install_requirements.sh

@@ -6,39 +6,7 @@
 # LICENSE file in the root directory of this source tree.
 
 set -x
-OS=$(uname)
 
-# install llava from the submodule. We can't do pip install llava because it is packaged incorrectly.
-if [[ $OS != "Darwin" ]];
-then
-    #This doesn't work for macos, on python 3.12, because torch 2.1.2 is missing.
-    pip install --force-reinstall -e examples/third-party/LLaVA
-else
-    # manually install dependencies
-    pip install tokenizers==0.15.1 sentencepiece==0.1.99    \
-        shortuuid accelerate==0.21.0 peft                   \
-        pydantic markdown2[all]  scikit-learn==1.2.2        \
-        requests httpx==0.24.0 uvicorn fastapi              \
-        einops==0.6.1 einops-exts==0.0.4 timm==0.6.13
-
-    pip install --force-reinstall -e examples/third-party/LLaVA --no-deps
-fi
-
-# not included in the pip install package, but needed in llava
-pip install protobuf
-
-# bitsandbytes depends on numpy 1.x, which is not compatible with numpy 2.x.
-# Reinstall bitsandbytes to make it compatible.
-pip install bitsandbytes -I
-
-# The deps of llava can have different versions than deps of ExecuTorch.
-# For example, torch version required from llava is older than ExecuTorch.
-# To make both work, recover ExecuTorch's original dependencies by rerunning
-# the install_requirements.sh. Notice this won't install executorch.
-bash -x ./install_requirements.sh --pybind xnnpack
-
-# Newer transformer (4.38) will give TypeError: LlavaLlamaForCausalLM.forward() got an unexpected keyword argument 'cache_position'
-pip install timm==0.6.13
-pip install transformers==4.37.2
+pip install transformers
 
 pip list