add oss model

Author: neuropilot-captain

examples/mediatek/CMakeLists.txt

@@ -93,6 +93,44 @@ if(${ANDROID})
         ${_common_compile_options}
     )
 
+    set(_mtk_oss_executor_runner__srcs ${_executor_runner__srcs})
+    list(
+        TRANSFORM
+        _mtk_oss_executor_runner__srcs
+        PREPEND
+        "${EXECUTORCH_SOURCE_DIR}/"
+    )
+    list(
+        FILTER
+        _mtk_oss_executor_runner__srcs
+        EXCLUDE REGEX
+        ".*executor_runner.cpp$"
+    )
+    list(
+        PREPEND
+        _mtk_oss_executor_runner__srcs
+        ${CMAKE_CURRENT_LIST_DIR}/executor_runner/mtk_oss_executor_runner.cpp
+    )
+
+    add_executable(mtk_oss_executor_runner ${_mtk_oss_executor_runner__srcs})
+
+    target_include_directories(mtk_oss_executor_runner
+        PUBLIC
+        ${_common_include_directories}
+        ${EXECUTORCH_ROOT}/cmake-android-out/third-party/gflags/include
+    )
+
+    target_link_libraries(mtk_oss_executor_runner
+        ${_executor_runner_libs}
+        executorch
+        neuron_backend
+        gflags
+    )
+    target_compile_options(mtk_oss_executor_runner
+        PUBLIC
+        ${_common_compile_options}
+    )
+
     set(_mtk_llama_executor_runner__srcs ${_mtk_executor_runner__srcs})
     list(
         FILTER

examples/mediatek/README.md

@@ -9,6 +9,8 @@ examples/mediatek
         ├── preformatter_templates    # Model specific prompt preformatter templates
         ├── prompts                   # Calibration Prompts
         ├── tokenizers_               # Model tokenizer scripts
+    ├── oss_utils                     # Utils for oss models
+├── eval_utils                        # Utils for eval oss models
 ├── model_export_scripts              # Model specifc export scripts
 ├── models                            # Model definitions
     ├── llm_models                    # LLM model definitions
@@ -44,6 +46,7 @@ pip3 install mtk_converter-8.8.0.dev20240723+public.d1467db9-cp310-cp310-manylin
 ```
 
 ## AoT Flow
+### llama
 ##### Note: Verify that localhost connection is available before running AoT Flow
 1. Exporting Models to `.pte`
 - In the `examples/mediatek directory`, run:
@@ -72,6 +75,14 @@ source shell_scripts/export_llama.sh <model_name> <num_chunks> <prompt_num_token
     - eg. For `llama3-8B-instruct`, embedding bin generated in `examples/mediatek/models/llm_models/weights/llama3-8B-instruct/`
     - AoT flow will take roughly 2.5 hours (114GB RAM for `num_chunks=4`) to complete (Results will vary by device/hardware configurations)
 
+### oss
+1. Exporting Model to `.pte`
+```bash
+bash shell_scripts/export_oss.sh <model_name>
+```
+- Argument Options:
+    - `model_name`: deeplabv3/edsr/inceptionv3/inceptionv4/mobilenetv2/mobilenetv3/resnet18/resnet50
+
 # Runtime
 ## Supported Chips
 
@@ -100,6 +111,13 @@ adb push <MODEL_NAME>.pte <PHONE_PATH, e.g. /data/local/tmp>
 
 Make sure to replace `<MODEL_NAME>` with the actual name of your model file. And, replace the `<PHONE_PATH>` with the desired detination on the device.
 
+##### Note: For oss models, please push additional files to your Android device
+```bash
+adb push mtk_oss_executor_runner <PHONE_PATH, e.g. /data/local/tmp>
+adb push input_list.txt <PHONE_PATH, e.g. /data/local/tmp>
+for i in input*bin; do adb push "$i" <PHONE_PATH, e.g. /data/local/tmp>; done;
+```
+
 ### Executing the Model
 
 Execute the model on your Android device by running:
@@ -111,3 +129,22 @@ adb shell "/data/local/tmp/mtk_executor_runner --model_path /data/local/tmp/<MOD
 In the command above, replace `<MODEL_NAME>` with the name of your model file and `<ITER_TIMES>` with the desired number of iterations to run the model.
 
 ##### Note: For llama models, please use `mtk_llama_executor_runner`. Refer to `examples/mediatek/executor_runner/run_llama3_sample.sh` for reference.
+##### Note: For oss models, please use `mtk_oss_executor_runner`.
+```bash
+adb shell "/data/local/tmp/mtk_oss_executor_runner --model_path /data/local/tmp/<MODEL_NAME>.pte --input_list /data/local/tmp/input_list.txt --output_folder /data/local/tmp/output_<MODEL_NAME>"
+adb pull "/data/local/tmp/output_<MODEL_NAME> ./"
+```
+
+### Check oss result on PC
+```bash
+python3 eval_utils/eval_oss_result.py --eval_type <eval_type> --target_f <golden_folder> --output_f <prediction_folder>
+```
+For example:
+```
+python3 eval_utils/eval_oss_result.py --eval_type piq --target_f edsr --output_f output_edsr
+```
+- Argument Options:
+    - `eval_type`: topk/piq/segmentation
+    - `target_f`: folder contain golden data files. file name is `golden_<data_idx>_0.bin`
+    - `output_f`: folder contain model output data files. file name is `output_<data_idx>_0.bin`
+

examples/mediatek/aot_utils/oss_utils/utils.py

@@ -0,0 +1,66 @@
+# Copyright (c) MediaTek Inc.
+# All rights reserved
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+import os
+from typing import Optional
+import numpy as np
+
+import torch
+from executorch import exir
+from executorch.backends.mediatek import NeuropilotPartitioner, NeuropilotQuantizer, Precision
+from torch.ao.quantization.quantize_pt2e import convert_pt2e, prepare_pt2e
+
+
+def build_executorch_binary(
+    model,
+    inputs,
+    file_name,
+    dataset,
+    quant_dtype: Optional[Precision] = None,
+):
+    if quant_dtype is not None:
+        quantizer = NeuropilotQuantizer()
+        quantizer.setup_precision(quant_dtype)
+        if quant_dtype not in Precision:
+            raise AssertionError(f"No support for Precision {quant_dtype}.")
+
+        captured_model = torch._export.capture_pre_autograd_graph(model, inputs)
+        annotated_model = prepare_pt2e(captured_model, quantizer)
+        print("Quantizing the model...")
+        # calibration
+        for data in dataset:
+            annotated_model(*data)
+        quantized_model = convert_pt2e(annotated_model, fold_quantize=False)
+        aten_dialect = torch.export.export(quantized_model, inputs)
+    else:
+        aten_dialect = torch.export.export(model, inputs)
+
+    from executorch.exir.program._program import to_edge_transform_and_lower
+
+    edge_compile_config = exir.EdgeCompileConfig(_check_ir_validity=False)
+    # skipped op names are used for deeplabV3 model
+    neuro_partitioner = NeuropilotPartitioner([], op_names_to_skip={'aten_convolution_default_106', 'aten_convolution_default_107'})
+    edge_prog = to_edge_transform_and_lower(
+            aten_dialect,
+            compile_config=edge_compile_config,
+            partitioner=[neuro_partitioner],
+    )
+
+    exec_prog = edge_prog.to_executorch(
+        config=exir.ExecutorchBackendConfig(extract_constant_segment=False)
+    )
+    with open(f"{file_name}.pte", "wb") as file:
+        file.write(exec_prog.buffer)
+
+
+def make_output_dir(path: str):
+    if os.path.exists(path):
+        for f in os.listdir(path):
+            os.remove(os.path.join(path, f))
+        os.removedirs(path)
+    os.makedirs(path)
+
+

examples/mediatek/eval_utils/eval_oss_result.py

@@ -0,0 +1,198 @@
+# Copyright (c) MediaTek Inc.
+# All rights reserved
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+import sys
+import os
+import piq
+import numpy as np
+import argparse
+import torch
+import json
+
+
+def check_data(target_f, predict_f):
+    target_files = os.listdir(target_f)
+    predict_files = os.listdir(predict_f)
+    if len(target_files) != len(predict_files):
+        raise RuntimeError("Data number in target folder and prediction folder must be same")
+
+    predict_set = set(predict_files)
+    for f in target_files:
+        # target file naming rule is golden_sampleId_outId.bin
+        # predict file naming rule is output_sampleId_outId.bin
+        pred_name = f.replace("golden", "output")
+        try:
+            predict_set.remove(pred_name)
+        except KeyError:
+            raise RuntimeError(f"Cannot find {pred_name} in {predict_f}")
+
+    if predict_set:
+        target_name = next(predict_set).replace("output", "golden")
+        raise RuntimeError(f"Cannot find {target_name} in {target_f}")
+
+
+def eval_topk(target_f, predict_f):
+    def solve(prob, target, k):
+        _, indices = torch.topk(prob, k=k, sorted=True)
+        golden = torch.reshape(target, [-1, 1])
+        correct = (golden == indices)
+        if torch.any(correct):
+            return 1
+        else:
+            return 0
+
+    target_files = os.listdir(target_f)
+
+    cnt10 = 0
+    cnt50 = 0
+    for target_name in target_files:
+        pred_name = target_name.replace("golden", "output")
+
+        pred_npy = np.fromfile(os.path.join(predict_f, pred_name), dtype=np.float32)
+        target_npy = np.fromfile(os.path.join(target_f, target_name), dtype=np.int64)[0]
+        cnt10 += solve(torch.from_numpy(pred_npy), torch.from_numpy(target_npy), 10)
+        cnt50 += solve(torch.from_numpy(pred_npy), torch.from_numpy(target_npy), 50)
+
+    print("Top10 acc:", cnt10 * 100.0 / len(target_files))
+    print("Top50 acc:", cnt50 * 100.0 / len(target_files))
+
+
+def eval_piq(target_f, predict_f):
+    target_files = os.listdir(target_f)
+    
+    psnr_list = []
+    ssim_list = []
+    for target_name in target_files:
+        pred_name = target_name.replace("golden", "output")
+        hr = np.fromfile(os.path.join(target_f, target_name), dtype=np.float32)
+        hr = hr.reshape((1,448,448,3))
+        hr = np.moveaxis(hr, 3, 1)
+        hr = torch.from_numpy(hr)
+
+        sr = np.fromfile(os.path.join(predict_f, pred_name), dtype=np.float32)
+        sr = sr.reshape((1,448,448,3))
+        sr = np.moveaxis(sr, 3, 1)
+        sr = torch.from_numpy(sr).clamp(0, 1)
+
+        psnr_list.append(piq.psnr(hr, sr))
+        ssim_list.append(piq.ssim(hr, sr))
+
+    avg_psnr = sum(psnr_list).item() / len(psnr_list)
+    avg_ssim = sum(ssim_list).item() / len(ssim_list)
+
+    print(f"Avg of PSNR is: {avg_psnr}")
+    print(f"Avg of SSIM is: {avg_ssim}")
+
+
+def eval_segmentation(target_f, predict_f):
+    classes = [
+        "Backround",
+        "Aeroplane",
+        "Bicycle",
+        "Bird",
+        "Boat",
+        "Bottle",
+        "Bus",
+        "Car",
+        "Cat",
+        "Chair",
+        "Cow",
+        "DiningTable",
+        "Dog",
+        "Horse",
+        "MotorBike",
+        "Person",
+        "PottedPlant",
+        "Sheep",
+        "Sofa",
+        "Train",
+        "TvMonitor",
+    ]
+
+    target_files = os.listdir(target_f)
+
+    def make_confusion(goldens, predictions, num_classes):
+        def histogram(golden, predict):
+            mask = golden < num_classes
+            hist = np.bincount(
+                num_classes * golden[mask].astype(int) + predict[mask],
+                minlength=num_classes**2,
+            ).reshape(num_classes, num_classes)
+            return hist
+
+        confusion = np.zeros((num_classes, num_classes))
+        for g, p in zip(goldens, predictions):
+            confusion += histogram(g.flatten(), p.flatten())
+
+        return confusion
+
+    pred_list = []
+    target_list = []
+    for target_name in target_files:
+        pred_name = target_name.replace("golden", "output")
+        target_npy = np.fromfile(os.path.join(target_f, target_name), dtype=np.uint8)
+        target_npy = target_npy.reshape((224, 224))
+        target_list.append(target_npy)
+
+        pred_npy = np.fromfile(os.path.join(predict_f, pred_name), dtype=np.float32)
+        pred_npy = pred_npy.reshape((224, 224, len(classes)))
+        pred_npy = pred_npy.argmax(2).astype(np.uint8)
+        pred_list.append(pred_npy)
+
+    eps = 1e-6
+    confusion = make_confusion(target_list, pred_list, len(classes))
+
+    pa = np.diag(confusion).sum() / (confusion.sum() + eps)
+    mpa = np.mean(np.diag(confusion) / (confusion.sum(axis=1) + eps))
+    iou = np.diag(confusion) / (
+        confusion.sum(axis=1) + confusion.sum(axis=0) - np.diag(confusion) + eps
+    )
+    miou = np.mean(iou)
+    cls_iou = dict(zip(classes, iou))
+
+    print(f"PA   : {pa}")
+    print(f"MPA  : {mpa}")
+    print(f"MIoU : {miou}")
+    print(f"CIoU : \n{json.dumps(cls_iou, indent=2)}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        "--target_f",
+        help="folder of target data",
+        type=str,
+        required=True,
+    )
+
+    parser.add_argument(
+        "--out_f",
+        help="folder of model prediction data",
+        type=str,
+        required=True,
+    )
+
+    parser.add_argument(
+        "--eval_type",
+        help="Choose eval type from: topk, piq, segmentation",
+        type=str,
+        choices=["topk", "piq", "segmentation"],
+        required=True,
+    )
+
+    args = parser.parse_args()
+
+    check_data(args.target_f, args.out_f)
+
+    if args.eval_type == 'topk':
+        eval_topk(args.target_f, args.out_f)
+    elif args.eval_type == 'piq':
+        eval_piq(args.target_f, args.out_f)
+    elif args.eval_type == 'segmentation':
+        eval_segmentation(args.target_f, args.out_f)
+
+