Clean up llava deps

Author: Guang Yang

.ci/scripts/test.sh

@@ -71,10 +71,9 @@ test_model() {
   if [[ "${MODEL_NAME}" == "llava" ]]; then
     # Install requirements for llava
     bash examples/models/llava/install_requirements.sh
-    STRICT="--no-strict"
   fi
   # python3 -m examples.portable.scripts.export --model_name="llama2" should works too
-  "${PYTHON_EXECUTABLE}" -m examples.portable.scripts.export --model_name="${MODEL_NAME}" "${STRICT}"
+  "${PYTHON_EXECUTABLE}" -m examples.portable.scripts.export --model_name="${MODEL_NAME}"
   run_portable_executor_runner
 }
 

examples/models/llava/README.md

@@ -7,14 +7,13 @@ In this example, we initiate the process of running multi modality through Execu
 Note that this folder does not host the pretrained LLava model.
 - To have Llava available, follow the [Install instructions](https://github.com/haotian-liu/LLaVA?tab=readme-ov-file#install) in the LLava github. Follow the licence in the specific repo when using L
 - Since the pytorch model version may not be updated, `cd executorch`, run `./install_requirements.sh`.
-- If there is numpy compatibility issue, run `pip install bitsandbytes -I`.
-- Alternatively, run `examples/models/llava_encoder/install_requirements.sh`, to replace the steps above.
-- Run `python3 -m examples.portable.scripts.export --model_name="llava_encoder"`. The llava_encoder.pte file will be generated.
-- Run `./cmake-out/executor_runner --model_path ./llava_encoder.pte` to verify the exported model with ExecuTorch runtime with portable kernels. Note that the portable kernels are not performance optimized. Please refer to other examples like those in llama2 folder for optimization.
+- Run `examples/models/llava/install_requirements.sh`, to install llava specific deps.
+- Run `python3 -m examples.portable.scripts.export --model_name="llava"`. The llava.pte file will be generated.
+- Run `./cmake-out/executor_runner --model_path ./llava.pte` to verify the exported model with ExecuTorch runtime with portable kernels. Note that the portable kernels are not performance optimized. Please refer to other examples like those in llama2 folder for optimization.
 
 ## TODO
 - Write the pipeline in cpp
   - Have image and text prompts as inputs.
   - Call image processing functions to preprocess the image tensor.
-  - Load the llava_encoder.pte model, run it using the image tensor.
+  - Load the llava.pte model, run it using the image tensor.
   - The output of the encoder can be combined with the prompt, as inputs to the llama model. Call functions in llama_runner.cpp to run the llama model and get outputs. The ExecuTorch end to end flow for the llama model is located at `examples/models/llama2`.

examples/models/llava/install_requirements.sh

@@ -7,6 +7,28 @@
 
 set -x
 
-pip install transformers accelerate
+# install llava from the submodule. We can't do pip install llava because it is packaged incorrectly.
+if [[ $OS != "Darwin" ]];
+then
+    #This doesn't work for macos, on python 3.12, because torch 2.1.2 is missing.
+    pip install --force-reinstall -e examples/third-party/LLaVA
+else
+    # manually install dependencies
+    pip install tokenizers==0.15.1 sentencepiece==0.1.99    \
+        shortuuid accelerate==0.21.0 peft                   \
+        pydantic markdown2[all]  scikit-learn==1.2.2        \
+        requests httpx==0.24.0 uvicorn fastapi              \
+        einops==0.6.1 einops-exts==0.0.4 timm==0.6.13
 
-pip list
+    pip install --force-reinstall -e examples/third-party/LLaVA --no-deps
+fi
+
+# not included in the pip install package, but needed in llava
+pip install protobuf
+pip install triton==3.0.0
+
+# The deps of llava can have different versions than deps of ExecuTorch.
+# For example, torch version required from llava is older than ExecuTorch.
+# To make both work, recover ExecuTorch's original dependencies by rerunning
+# the install_requirements.sh. Notice this won't install executorch.
+bash -x ./install_requirements.sh --pybind xnnpack

examples/models/llava/model.py

@@ -10,7 +10,8 @@
 
 import re
 
-from typing import Any, Dict, Optional
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional
 
 import requests
 import torch
@@ -271,6 +272,54 @@ def __init__(self, use_sdpa_with_kv_cache_op=True):
         self.input = None
         self.resized_image = None
 
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        images: Optional[torch.FloatTensor] = None,
+        image_sizes: Optional[List[List[int]]] = None,
+        return_dict: Optional[bool] = None,
+        cache_position: Optional[torch.LongTensor] = None,
+    ):
+        """
+        An adapter to the llava_llama.forward(), making it compatible with latest HF interface.
+        """
+        # Do not pass 'cache_position' down to forward() as this old third-party llava can not recongize it.
+        return self.model.forward(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            labels=labels,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+    @torch.no_grad()
+    def generate(
+        self,
+        inputs: Optional[torch.Tensor] = None,
+        images: Optional[torch.Tensor] = None,
+        image_sizes: Optional[torch.Tensor] = None,
+        **kwargs,
+    ):
+        """
+        A adapter to the llava_llama.generate(), make it compatible with latest HF interface.
+        """
+        return self.model.generate(
+            inputs=inputs, images=images, image_sizes=image_sizes, **kwargs
+        )
+
     def get_eager_model(self):
         model = Llava(
             self.model,

examples/models/llava/test/test_llava.py

@@ -47,7 +47,7 @@ def test_generated_output(self):
         # source of truth, using HF llava
         preprocessed = self.llava.image_preprocess(self.resized)
         with torch.inference_mode():
-            output_ids = self.llava_model.model.generate(
+            output_ids = self.llava_model.generate(
                 self.llava_model.input_ids,
                 pixel_values=preprocessed,
                 do_sample=False,