Fix reporting backends and dtyep to benchmark results

Author: Guang Yang

.ci/scripts/test_llama.sh

@@ -171,7 +171,7 @@ else
 fi
 
 # Check dtype.
-EXPORTED_MODEL_NAME="llama2"
+EXPORTED_MODEL_NAME="tinyllama_${MODE}_${DTYPE}"
 if [[ "${DTYPE}" == "fp16" ]]; then
   EXPORTED_MODEL_NAME="${EXPORTED_MODEL_NAME}_h"
 elif [[ "${DTYPE}" == "bf16" ]]; then

.ci/scripts/test_model.sh

@@ -152,32 +152,34 @@ test_model_with_qnn() {
 
   export LD_LIBRARY_PATH=$QNN_SDK_ROOT/lib/x86_64-linux-clang/
   export PYTHONPATH=$EXECUTORCH_ROOT/..
+  # QNN support fp16 only
+  DTYPE=fp16
 
   if [[ "${MODEL_NAME}" == "dl3" ]]; then
     EXPORT_SCRIPT=deeplab_v3
-    EXPORTED_MODEL_NAME=dlv3_qnn.pte
+    EXPORTED_MODEL_NAME=deeplab-v3_qnn_${DTYPE}.pte
   elif [[ "${MODEL_NAME}" == "mv3" ]]; then
     EXPORT_SCRIPT=mobilenet_v3
-    EXPORTED_MODEL_NAME=mv3_qnn.pte
+    EXPORTED_MODEL_NAME=mobilenet-v3_qnn_${DTYPE}.pte
   elif [[ "${MODEL_NAME}" == "mv2" ]]; then
     EXPORT_SCRIPT=mobilenet_v2
-    EXPORTED_MODEL_NAME=mv2_qnn.pte
+    EXPORTED_MODEL_NAME=mobilenet-v2_qnn_${DTYPE}.pte
   elif [[ "${MODEL_NAME}" == "ic4" ]]; then
     EXPORT_SCRIPT=inception_v4
-    EXPORTED_MODEL_NAME=ic4_qnn.pte
+    EXPORTED_MODEL_NAME=inception-v4_qnn_${DTYPE}.pte
   elif [[ "${MODEL_NAME}" == "ic3" ]]; then
     EXPORT_SCRIPT=inception_v3
-    EXPORTED_MODEL_NAME=ic3_qnn.pte
+    EXPORTED_MODEL_NAME=inception-v3_qnn_${DTYPE}.pte
   elif [[ "${MODEL_NAME}" == "vit" ]]; then
     EXPORT_SCRIPT=torchvision_vit
-    EXPORTED_MODEL_NAME=vit_qnn.pte
+    EXPORTED_MODEL_NAME=torchvision-vit_qnn_${DTYPE}.pte
   fi
 
   # Use SM8450 for S22, SM8550 for S23, and SM8560 for S24
   # TODO(guangyang): Make QNN chipset matches the target device
   QNN_CHIPSET=SM8450
 
-  "${PYTHON_EXECUTABLE}" -m examples.qualcomm.scripts.${EXPORT_SCRIPT} -b ${CMAKE_OUTPUT_DIR} -m ${QNN_CHIPSET} --compile_only
+  "${PYTHON_EXECUTABLE}" -m examples.qualcomm.scripts.${EXPORT_SCRIPT} -b ${CMAKE_OUTPUT_DIR} -m ${QNN_CHIPSET} -o ${EXPORTED_MODEL_NAME} --compile_only
   EXPORTED_MODEL=./${EXPORT_SCRIPT}/${EXPORTED_MODEL_NAME}
 }
 

examples/apple/coreml/scripts/export.py

@@ -110,9 +110,9 @@ def export_lowered_module_to_executorch_program(lowered_module, example_inputs):
     return exec_prog
 
 
-def save_executorch_program(exec_prog, model_name, compute_unit):
+def save_executorch_program(exec_prog, model_name, compute_unit, precision):
     buffer = exec_prog.buffer
-    filename = f"{model_name}_coreml_{compute_unit}.pte"
+    filename = f"{model_name}_coreml_{compute_unit}_{precision}.pte"
     print(f"Saving exported program to {filename}")
     with open(filename, "wb") as file:
         file.write(buffer)
@@ -189,7 +189,9 @@ def main():
             example_inputs,
         )
 
-    save_executorch_program(exec_program, args.model_name, args.compute_unit)
+    save_executorch_program(
+        exec_program, args.model_name, args.compute_unit, args.compute_precision
+    )
     generate_etrecord(f"{args.model_name}_coreml_etrecord.bin", edge_copy, exec_program)
 
     if args.save_processed_bytes and lowered_module is not None:

examples/qualcomm/scripts/deeplab_v3.py

@@ -81,7 +81,7 @@ def main(args):
             data_size=data_num, dataset_dir=args.artifact, download=args.download
         )
 
-    pte_filename = "dlv3_qnn"
+    pte_filename = "dlv3_qnn" if not args.output_filename else args.output_filename
     instance = DeepLabV3ResNet101Model()
 
     build_executorch_binary(
@@ -195,6 +195,14 @@ def post_process():
         default=False,
     )
 
+    parser.add_argument(
+        "-o",
+        "--output_filename",
+        help="name of the output pte file",
+        type=str,
+        required=False,
+    )
+
     args = parser.parse_args()
     try:
         main(args)

examples/qualcomm/scripts/edsr.py

@@ -107,7 +107,7 @@ def main(args):
     )
 
     inputs, targets, input_list = dataset.lr, dataset.hr, dataset.get_input_list()
-    pte_filename = "edsr_qnn"
+    pte_filename = "edsr_qnn" if not args.output_filename else args.output_filename
     instance = EdsrModel()
 
     build_executorch_binary(
@@ -217,6 +217,14 @@ def post_process():
         default=False,
     )
 
+    parser.add_argument(
+        "-o",
+        "--output_filename",
+        help="name of the output pte file",
+        type=str,
+        required=False,
+    )
+
     args = parser.parse_args()
     try:
         main(args)

examples/qualcomm/scripts/inception_v3.py

@@ -46,7 +46,7 @@ def main(args):
             image_shape=(256, 256),
             crop_size=224,
         )
-    pte_filename = "ic3_qnn"
+    pte_filename = "ic3_qnn" if not args.output_filename else args.output_filename
     instance = InceptionV3Model()
     build_executorch_binary(
         instance.get_eager_model().eval(),
@@ -125,6 +125,14 @@ def main(args):
         type=str,
     )
 
+    parser.add_argument(
+        "-o",
+        "--output_filename",
+        help="name of the output pte file",
+        type=str,
+        required=False,
+    )
+
     args = parser.parse_args()
     try:
         main(args)

examples/qualcomm/scripts/inception_v4.py

@@ -45,7 +45,7 @@ def main(args):
             data_size=data_num,
             image_shape=(299, 299),
         )
-    pte_filename = "ic4_qnn"
+    pte_filename = "ic4_qnn" if not args.output_filename else args.output_filename
     instance = InceptionV4Model()
     build_executorch_binary(
         instance.get_eager_model().eval(),
@@ -124,6 +124,14 @@ def main(args):
         type=str,
     )
 
+    parser.add_argument(
+        "-o",
+        "--output_filename",
+        help="name of the output pte file",
+        type=str,
+        required=False,
+    )
+
     args = parser.parse_args()
     try:
         main(args)

examples/qualcomm/scripts/mobilenet_v2.py

@@ -46,7 +46,7 @@ def main(args):
             image_shape=(256, 256),
             crop_size=224,
         )
-    pte_filename = "mv2_qnn"
+    pte_filename = "mv2_qnn" if not args.output_filename else args.output_filename
     instance = MV2Model()
     build_executorch_binary(
         instance.get_eager_model().eval(),
@@ -125,6 +125,14 @@ def main(args):
         type=str,
     )
 
+    parser.add_argument(
+        "-o",
+        "--output_filename",
+        help="name of the output pte file",
+        type=str,
+        required=False,
+    )
+
     args = parser.parse_args()
     try:
         main(args)

examples/qualcomm/scripts/mobilenet_v3.py

@@ -45,7 +45,7 @@ def main(args):
             image_shape=(256, 256),
             crop_size=224,
         )
-    pte_filename = "mv3_qnn"
+    pte_filename = "mv3_qnn" if not args.output_filename else args.output_filename
     instance = MV3Model()
     build_executorch_binary(
         instance.get_eager_model().eval(),
@@ -123,6 +123,14 @@ def main(args):
         type=str,
     )
 
+    parser.add_argument(
+        "-o",
+        "--output_filename",
+        help="name of the output pte file",
+        type=str,
+        required=False,
+    )
+
     args = parser.parse_args()
     try:
         main(args)

examples/qualcomm/scripts/torchvision_vit.py

@@ -38,7 +38,7 @@ def main(args):
             crop_size=224,
         )
 
-    pte_filename = "vit_qnn"
+    pte_filename = "vit_qnn" if not args.output_filename else args.output_filename
     instance = TorchVisionViTModel()
     build_executorch_binary(
         instance.get_eager_model().eval(),
@@ -113,6 +113,14 @@ def main(args):
         type=str,
     )
 
+    parser.add_argument(
+        "-o",
+        "--output_filename",
+        help="name of the output pte file",
+        type=str,
+        required=False,
+    )
+
     args = parser.parse_args()
     try:
         main(args)