MTK build test

backends/mediatek/CMakeLists.txt

@@ -25,10 +25,10 @@ include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/runtime/include)
 
 # targets
 add_library(neuron_backend SHARED)
+target_compile_options(neuron_backend PRIVATE "-frtti" "-fexceptions")
 target_link_libraries(neuron_backend
     PRIVATE
     executorch_no_prim_ops
-    portable_ops_lib
     android
     log
     ${NEURON_BUFFER_ALLOCATOR_LIB}

backends/mediatek/runtime/include/NeuronLog.h

@@ -8,7 +8,7 @@
 
 #pragma once
 
-#include <api/NeuronAdapter.h>
+#include "api/NeuronAdapter.h"
 
 #include <android/log.h>
 #include <sys/system_properties.h>

build/build_android_llm_demo.sh

@@ -30,7 +30,7 @@ build_android_native_library() {
   cmake . -DCMAKE_INSTALL_PREFIX="${CMAKE_OUT}" \
     -DCMAKE_TOOLCHAIN_FILE="${ANDROID_NDK}/build/cmake/android.toolchain.cmake" \
     -DANDROID_ABI="${ANDROID_ABI}" \
-    -DANDROID_PLATFORM=android-23 \
+    -DANDROID_PLATFORM=android-26 \
     -DEXECUTORCH_ENABLE_LOGGING=ON \
     -DEXECUTORCH_LOG_LEVEL=Info \
     -DEXECUTORCH_BUILD_XNNPACK=ON \
@@ -42,36 +42,44 @@ build_android_native_library() {
     -DEXECUTORCH_BUILD_KERNELS_OPTIMIZED=ON \
     -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON \
     -DEXECUTORCH_BUILD_KERNELS_CUSTOM=ON \
+    -DEXECUTORCH_BUILD_NEURON=ON \
+    -DNEURON_BUFFER_ALLOCATOR_LIB="$NEURON_BUFFER_ALLOCATOR_LIB" \
     -DEXECUTORCH_BUILD_QNN="${EXECUTORCH_BUILD_QNN}" \
     -DQNN_SDK_ROOT="${QNN_SDK_ROOT}" \
-    -DCMAKE_BUILD_TYPE=Release \
+    -DCMAKE_BUILD_TYPE=RelWithDebInfo \
     -B"${CMAKE_OUT}"
 
   if [ "$(uname)" == "Darwin" ]; then
     CMAKE_JOBS=$(( $(sysctl -n hw.ncpu) - 1 ))
   else
     CMAKE_JOBS=$(( $(nproc) - 1 ))
   fi
-  cmake --build "${CMAKE_OUT}" -j "${CMAKE_JOBS}" --target install --config Release
+  cmake --build "${CMAKE_OUT}" -j "${CMAKE_JOBS}" --target install --config RelWithDebInfo
 
   cmake extension/android \
     -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake \
     -DANDROID_ABI="${ANDROID_ABI}" \
-    -DANDROID_PLATFORM=android-23 \
+    -DANDROID_PLATFORM=android-26 \
     -DCMAKE_INSTALL_PREFIX="${CMAKE_OUT}" \
     -DEXECUTORCH_ENABLE_LOGGING=ON \
     -DEXECUTORCH_LOG_LEVEL=Info \
+    -DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=BOTH \
+    -DNEURON_BUFFER_ALLOCATOR_LIB="$NEURON_BUFFER_ALLOCATOR_LIB" \
     -DEXECUTORCH_BUILD_KERNELS_CUSTOM=ON \
     -DEXECUTORCH_BUILD_LLAMA_JNI=ON \
-    -DCMAKE_BUILD_TYPE=Release \
+    -DCMAKE_BUILD_TYPE=RelWithDebInfo \
     -B"${CMAKE_OUT}"/extension/android
 
-  cmake --build "${CMAKE_OUT}"/extension/android -j "${CMAKE_JOBS}" --config Release
+  cmake --build "${CMAKE_OUT}"/extension/android -j "${CMAKE_JOBS}" --config RelWithDebInfo
 
   # Copy artifacts to ABI specific directory
   mkdir -p "${BUILD_AAR_DIR}/jni/${ANDROID_ABI}"
   cp "${CMAKE_OUT}"/extension/android/*.so "${BUILD_AAR_DIR}/jni/${ANDROID_ABI}/"
 
+  cp "${CMAKE_OUT}"/backends/mediatek/libneuron_backend.so ${BUILD_AAR_DIR}/jni/${ANDROID_ABI}/
+  cp /Users/cmodi/Documents/ai/clean/executorch/backends/mediatek/libneuron_buffer_allocator.so ${BUILD_AAR_DIR}/jni/${ANDROID_ABI}/
+  cp /Users/cmodi/Documents/ai/clean/executorch/backends/mediatek/libneuronusdk_adapter.mtk.so ${BUILD_AAR_DIR}/jni/${ANDROID_ABI}/
+
   # Copy QNN related so library
   if [ -n "$QNN_SDK_ROOT" ] && [ "$ANDROID_ABI" == "arm64-v8a" ]; then
     cp "${CMAKE_OUT}"/lib/libqnn_executorch_backend.so "${BUILD_AAR_DIR}/jni/${ANDROID_ABI}/"
@@ -97,7 +105,7 @@ build_aar() {
   find jni -type f -name "libexecutorch_jni.so" -exec bash -c 'mv "$1" "${1/_jni/}"' bash {} \;
   # Zip all necessary files into the AAR file
   zip -r executorch.aar libs jni/*/libexecutorch.so jni/*/libqnn*.so jni/*/libQnn*.so AndroidManifest.xml
-  zip -r executorch-llama.aar libs jni/*/libexecutorch.so jni/*/libqnn*.so jni/*/libQnn*.so AndroidManifest.xml
+  zip -r executorch-llama.aar libs jni/*/libexecutorch.so jni/*/libqnn*.so jni/*/libQnn*.so jni/*/libneuron_backend.so jni/*/libneuron_buffer_allocator.so jni/*/libneuronusdk_adapter.mtk.so AndroidManifest.xml
   popd
 }
 
@@ -143,6 +151,7 @@ BUILD_AAR_DIR="$(mktemp -d)"
 export BUILD_AAR_DIR
 if [ -z "$ANDROID_ABIS" ]; then
   ANDROID_ABIS=("arm64-v8a" "x86_64")
+  ANDROID_ABIS=("arm64-v8a")
 fi
 export ANDROID_ABIS
 

build/executorch-config.cmake

@@ -41,6 +41,7 @@ set(lib_list
     ${FLATCCRT_LIB}
     coremldelegate
     mpsdelegate
+    neuron_backend
     qnn_executorch_backend
     portable_ops_lib
     extension_module

examples/demo-apps/android/LlamaDemo/app/build.gradle.kts

@@ -57,7 +57,7 @@ dependencies {
   implementation("androidx.constraintlayout:constraintlayout:2.2.0-alpha12")
   implementation("com.facebook.fbjni:fbjni:0.5.1")
   implementation("com.google.code.gson:gson:2.8.6")
-  implementation(files("libs/executorch-llama.aar"))
+  implementation(files("libs/executorch-llama-mtk31.aar"))
   implementation("com.google.android.material:material:1.12.0")
   implementation("androidx.activity:activity:1.9.0")
   testImplementation("junit:junit:4.13.2")

examples/demo-apps/android/LlamaDemo/app/src/main/AndroidManifest.xml

@@ -36,6 +36,30 @@
             android:name="libcdsprpc.so"
             android:required="false" />
 
+        <uses-native-library
+            android:name="libapuwareutils_v2.mtk.so"
+            android:required="false" />
+
+        <uses-native-library
+            android:name="libapuwareapusys_v2.mtk.so"
+            android:required="false" />
+
+        <uses-native-library
+            android:name="libnir_neon_driver_ndk.mtk.so"
+            android:required="false" />
+
+        <uses-native-library
+            android:name="libnir_neon_driver_ndk.mtk.vndk.so"
+            android:required="false" />
+
+        <uses-native-library
+            android:name="libcmdl_ndk.mtk.vndk.so"
+            android:required="false" />
+
+        <uses-native-library
+            android:name="libcmdl_ndk.mtk.so"
+            android:required="false" />
+
         <activity
             android:name=".MainActivity"
             android:exported="true"

examples/demo-apps/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/MainActivity.java

@@ -126,6 +126,7 @@ private void setLocalModel(String modelPath, String tokenizerPath, float tempera
             modelPath,
             tokenizerPath,
             temperature);
+    ETLogging.getInstance().log("ModelType is: " + mCurrentSettingsFields.getModelType());
     int loadResult = mModule.load();
     long loadDuration = System.currentTimeMillis() - runStartTime;
     String modelLoadError = "";
@@ -156,11 +157,11 @@ private void setLocalModel(String modelPath, String tokenizerPath, float tempera
               + " sec."
               + " You can send text or image for inference";
 
-      if (mCurrentSettingsFields.getModelType() == ModelType.LLAVA_1_5) {
+      /*if (mCurrentSettingsFields.getModelType() == ModelType.LLAVA_1_5) {
         ETLogging.getInstance().log("Llava start prefill prompt");
         startPos = mModule.prefillPrompt(PromptFormat.getLlavaPresetPrompt(), 0, 1, 0);
         ETLogging.getInstance().log("Llava completes prefill prompt");
-      }
+      }*/
     }
 
     Message modelLoadedMessage = new Message(modelInfo, false, MessageType.SYSTEM, 0);
@@ -226,6 +227,9 @@ protected void onCreate(Bundle savedInstanceState) {
 
     try {
       Os.setenv("ADSP_LIBRARY_PATH", getApplicationInfo().nativeLibraryDir, true);
+      Os.setenv("LD_LIBRARY_PATH", getApplicationInfo().nativeLibraryDir, true);
+      ETLogging.getInstance().log("cmodiiiii ADSP_LIBRARY_PATH is: " + Os.getenv("ADSP_LIBRARY_PATH"));
+      ETLogging.getInstance().log("cmodiiiii LD_LIBRARY_PATH is: " + Os.getenv("LD_LIBRARY_PATH"));
     } catch (ErrnoException e) {
       finish();
     }
@@ -566,7 +570,7 @@ private void showMediaPreview(List<Uri> uris) {
 
     // For LLava, we want to call prefill_image as soon as an image is selected
     // Llava only support 1 image for now
-    if (mCurrentSettingsFields.getModelType() == ModelType.LLAVA_1_5) {
+/*    if (mCurrentSettingsFields.getModelType() == ModelType.LLAVA_1_5) {
       List<ETImage> processedImageList = getProcessedImagesForModel(mSelectedImageUri);
       if (!processedImageList.isEmpty()) {
         mMessageAdapter.add(
@@ -588,7 +592,7 @@ private void showMediaPreview(List<Uri> uris) {
             };
         executor.execute(runnable);
       }
-    }
+    }*/
   }
 
   private void addSelectedImagesToChatThread(List<Uri> selectedImageUri) {
@@ -689,24 +693,23 @@ public void run() {
                         }
                       });
                   long generateStartTime = System.currentTimeMillis();
-                  if (ModelUtils.getModelCategory(mCurrentSettingsFields.getModelType())
+                 /* if (ModelUtils.getModelCategory(mCurrentSettingsFields.getModelType())
                       == ModelUtils.VISION_MODEL) {
                     mModule.generateFromPos(
                         mCurrentSettingsFields.getFormattedSystemAndUserPrompt(rawPrompt),
                         ModelUtils.VISION_MODEL_SEQ_LEN,
                         startPos,
                         MainActivity.this,
                         false);
-                  } else {
+                  } else {*/
                     String finalPrompt =
                         getTotalFormattedPrompt(getConversationHistory(), rawPrompt);
                     ETLogging.getInstance().log("Running inference.. prompt=" + finalPrompt);
                     mModule.generate(
                         finalPrompt,
                         (int) (finalPrompt.length() * 0.75) + 64,
-                        MainActivity.this,
-                        false);
-                  }
+                        MainActivity.this);
+                  //}
 
                   long generateDuration = System.currentTimeMillis() - generateStartTime;
                   mResultMessage.setTotalGenerationTime(generateDuration);

examples/demo-apps/android/LlamaDemo/app/src/main/java/com/example/executorchllamademo/SettingsActivity.java

@@ -46,6 +46,8 @@ public class SettingsActivity extends AppCompatActivity {
 
   private DemoSharedPreferences mDemoSharedPreferences;
   public static double TEMPERATURE_MIN_VALUE = 0.0;
+  public static String MODEL_PATH="/data/local/tmp/et-mtk/llama3";
+  //public static String MODEL_PATH="/data/local/tmp/llama";
 
   @Override
   protected void onCreate(Bundle savedInstanceState) {
@@ -286,7 +288,7 @@ private void showInvalidPromptDialog() {
   }
 
   private void setupModelSelectorDialog() {
-    String[] pteFiles = listLocalFile("/data/local/tmp/llama/", ".pte");
+    String[] pteFiles = listLocalFile(MODEL_PATH, ".pte");
     AlertDialog.Builder modelPathBuilder = new AlertDialog.Builder(this);
     modelPathBuilder.setTitle("Select model path");
 
@@ -342,7 +344,7 @@ private void setupModelTypeSelectorDialog() {
   }
 
   private void setupTokenizerSelectorDialog() {
-    String[] binFiles = listLocalFile("/data/local/tmp/llama/", ".bin");
+    String[] binFiles = listLocalFile(MODEL_PATH, ".bin");
     String[] tokenizerFiles = new String[binFiles.length];
     System.arraycopy(binFiles, 0, tokenizerFiles, 0, binFiles.length);
     AlertDialog.Builder tokenizerPathBuilder = new AlertDialog.Builder(this);

examples/mediatek/executor_runner/llama_runner/LlamaModelChunk.cpp

@@ -73,16 +73,26 @@ size_t LlamaModelChunk::GetExpectedOutputCount() const {
 }
 
 void LlamaModelChunk::Initialize() {
+  ET_LOG(Info, "cmodiii in LlamaModelChunk::Initialize");
   LoadModels();
+  ET_LOG(Info, "cmodiii after LoadModels");
   GetModelIoInfo();
+  ET_LOG(Info, "cmodiii after GetModelIoInfo");
   CheckIoCount();
+  ET_LOG(Info, "cmodiii after CheckIoCount");
   PrepareCacheIOs();
+  ET_LOG(Info, "cmodiii after PrepareCacheIOs");
   AllocateIoBuffers();
+  ET_LOG(Info, "cmodiii after AllocateIoBuffers");
   InitMaskBuilder();
+  ET_LOG(Info, "cmodiii after InitMaskBuilder");
   InitCache();
+  ET_LOG(Info, "cmodiii after InitCache");
 
   SetBackendInputs();
+  ET_LOG(Info, "cmodiii after SetBackendInputs");
   SetBackendOutputs();
+  ET_LOG(Info, "cmodiii after SetBackendOutputs");
   mIsInitialized = true;
 }
 

examples/mediatek/executor_runner/llama_runner/LlamaRuntime.cpp

@@ -28,14 +28,21 @@ void LlamaRuntime::Initialize(
   const size_t numCache = 2 * modelOptions.num_layer / numChunk;
   ET_CHECK_MSG(numChunk > 0, "No model to initialize");
 
+  ET_LOG(Info, "cmodiii 1");
+  ET_LOG(Info, "cmodiii numChunk = %zu", numChunk);
+  ET_LOG(Info, "cmodiii numCache = %zu", numCache);
+
   // Initialize rotary embedding master lookup table
   const size_t rotEmbDim = modelOptions.hidden_size / modelOptions.num_head;
+  ET_LOG(Info, "cmodiii 2");
   mRotEmbMasterLut = std::make_unique<llm_helper::RotaryEmbeddingMasterLut>(
       modelOptions.rot_emb_type,
       modelOptions.max_token_length,
       rotEmbDim,
       modelOptions.rot_emb_base);
+  ET_LOG(Info, "cmodiii 3");
   mRotEmbMasterLut->generate();
+  ET_LOG(Info, "cmodiii 4");
 
   constexpr size_t numRotEmbInputs = 1;
   const bool usePromptModel = !modelPaths.prompt_model_paths.empty();
@@ -50,8 +57,10 @@ void LlamaRuntime::Initialize(
         return;
       modelPathMap[batchSize] = modelPaths[chunkIdx];
     };
+    ET_LOG(Info, "cmodiii 5");
     addModelPath(
         modelPaths.prompt_model_paths, modelOptions.prompt_token_batch_size);
+    ET_LOG(Info, "cmodiii 6");
     addModelPath(modelPaths.gen_model_paths, 1);
     auto llamaChunk = std::make_unique<LlamaModelChunk>(
         modelPathMap,
@@ -60,18 +69,43 @@ void LlamaRuntime::Initialize(
         numCache,
         numRotEmbInputs,
         mRotEmbMasterLut.get());
+    ET_LOG(Info, "cmodiii 7");
+    if(llamaChunk.get() == nullptr) {
+      ET_LOG(Info, "cmodiii llamaChunk is null");
+    } else {
+      ET_LOG(Info, "cmodiii llamaChunk is not null");
+    }
+
     mLlamaModelChunks.push_back(std::move(llamaChunk));
+
+    if(mLlamaModelChunks.empty()) {
+      ET_LOG(Info, "cmodiii mLlamaModelChunks is empty");
+    } else {
+      ET_LOG(Info, "cmodiii mLlamaModelChunks is not empty");
+    }
+
+    ET_LOG(Info, "cmodiii 8");
   }
 
   for (size_t i = 0; i < numChunk; i++) {
     auto& modelChunk = mLlamaModelChunks[i];
+    if(modelChunk.get() == nullptr) {
+      ET_LOG(Info, "cmodiii modelChunk is null");
+    } else {
+      ET_LOG(Info, "cmodiii modelChunk is not null");
+    }
+    ET_LOG(Info, "cmodiii 9");
     if (i > 0) {
       const auto& prevModelChunk = mLlamaModelChunks[i - 1];
+      ET_LOG(Info, "cmodiii 9A");
       modelChunk->SetInputBuffer(prevModelChunk->GetOutputBuffer());
+      ET_LOG(Info, "cmodiii 10");
     }
     modelChunk->Initialize();
+    ET_LOG(Info, "cmodiii 11");
     // modelChunk->LogIoSummary();
   }
+  ET_LOG(Info, "cmodiii 12");
 
   // NOTE: Token embedding type here is assumed to follow the model input
   // embedding type.
@@ -80,9 +114,13 @@ void LlamaRuntime::Initialize(
       modelOptions.model_input_type,
       modelOptions.hidden_size);
 
+  ET_LOG(Info, "cmodiii 13");
+
   // Link first chunk emb input to token emb lut output
   const auto& tokenEmbInput = mLlamaModelChunks.front()->GetInputBuffer();
+  ET_LOG(Info, "cmodiii 14");
   mTokenEmbLut->setOutput(tokenEmbInput.data, tokenEmbInput.nbytes);
+  ET_LOG(Info, "cmodiii 15");
 }
 
 void LlamaRuntime::Release() {
@@ -201,4 +239,4 @@ const LlamaModelOptions& LlamaRuntime::GetModelOptions() const {
   return mModelOptions;
 }
 
-} // namespace torch::executor
+} // namespace torch::executor