Skip to content

Commit 7b4be54

Browse files
winskuo-quicwinskuo-quic
authored
Qualcomm AI Engine Direct - Use AIHub's context binary file for Stable Diffusion (#4836)
Summary: - Add script for the export and runtime of AIHUB Stable Diffusion. - Add AIHUB Stable Diffusion runner - Add README tutorial
1 parent 48f4eee commit 7b4be54

File tree

10 files changed

+1514
-0
lines changed

10 files changed

+1514
-0
lines changed

‎backends/qualcomm/tests/test_qnn_delegate.py

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1998,6 +1998,55 @@ def test_llama3_8b(self):
19981998
model_out = msg["result"]
19991999
self.assertTrue(model_out.startswith(prompt))
20002000

2001+
def test_stable_diffusion(self):
2002+
if not self.required_envs():
2003+
self.skipTest("missing required envs")
2004+
2005+
prompt = "a photo of an astronaut riding a horse on mars"
2006+
cmds = [
2007+
"python",
2008+
f"{self.executorch_root}/examples/qualcomm/qaihub_scripts/stable_diffusion/qaihub_stable_diffusion.py",
2009+
"--artifact",
2010+
self.artifact_dir,
2011+
"--build_folder",
2012+
self.build_folder,
2013+
"--device",
2014+
self.device,
2015+
"--model",
2016+
self.model,
2017+
"--text_encoder_bin",
2018+
f"{self.artifact_dir}/text_encoder.serialized.bin",
2019+
"--unet_bin",
2020+
f"{self.artifact_dir}/unet.serialized.bin",
2021+
"--vae_bin",
2022+
f"{self.artifact_dir}/vae.serialized.bin",
2023+
"--vocab_json",
2024+
f"{self.artifact_dir}/vocab.json",
2025+
"--num_time_steps",
2026+
"20",
2027+
"--ip",
2028+
self.ip,
2029+
"--port",
2030+
str(self.port),
2031+
"--prompt",
2032+
f"{prompt}",
2033+
"--fix_latents",
2034+
]
2035+
if self.host:
2036+
cmds.extend(["--host", self.host])
2037+
2038+
p = subprocess.Popen(cmds, stdout=subprocess.DEVNULL)
2039+
with Listener((self.ip, self.port)) as listener:
2040+
conn = listener.accept()
2041+
p.communicate()
2042+
msg = json.loads(conn.recv())
2043+
if "Error" in msg:
2044+
self.fail(msg["Error"])
2045+
else:
2046+
# For the default settings and prompt, the expected results will be {PSNR: 23.258, SSIM: 0.852}
2047+
self.assertGreaterEqual(msg["PSNR"], 20)
2048+
self.assertGreaterEqual(msg["SSIM"], 0.8)
2049+
20012050

20022051
class TestExampleScript(TestQNN):
20032052
def required_envs(self, conditions=None) -> bool:

‎examples/qualcomm/CMakeLists.txt

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,3 +81,8 @@ add_subdirectory(
8181
add_subdirectory(
8282
${CMAKE_CURRENT_SOURCE_DIR}/qaihub_scripts/llama
8383
)
84+
85+
# build qaihub_stable_diffusion_runner
86+
add_subdirectory(
87+
${CMAKE_CURRENT_SOURCE_DIR}/qaihub_scripts/stable_diffusion
88+
)

‎examples/qualcomm/qaihub_scripts/stable_diffusion/CMakeLists.txt

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
# Copyright (c) Qualcomm Innovation Center, Inc.
2+
# All rights reserved
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
# preprocess qaihub_stable_diffusion_runner_src files
8+
set(_qaihub_stable_diffusion_runner__srcs
9+
${CMAKE_CURRENT_LIST_DIR}/qaihub_stable_diffusion_runner.cpp
10+
${CMAKE_CURRENT_LIST_DIR}/runner/runner.cpp
11+
${CMAKE_CURRENT_LIST_DIR}/runner/runner.h
12+
)
13+
14+
# build qaihub_stable_diffusion_runner
15+
add_executable(qaihub_stable_diffusion_runner ${_qaihub_stable_diffusion_runner__srcs})
16+
target_include_directories(qaihub_stable_diffusion_runner
17+
PUBLIC ${_common_include_directories}
18+
)
19+
target_link_libraries(qaihub_stable_diffusion_runner
20+
qnn_executorch_backend
21+
executorch_no_prim_ops
22+
extension_data_loader
23+
extension_module
24+
gflags
25+
)
26+
target_compile_options(qaihub_stable_diffusion_runner PUBLIC ${_common_compile_options})

‎examples/qualcomm/qaihub_scripts/stable_diffusion/README.md

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
# Summary
2+
3+
## Overview
4+
This file provides you the instructions to run Stable-Diffusion-v2.1 with different parameters via Qualcomm HTP backend. We will demonstrate how to run Stable Diffusion v2.1 on mobile devices using context binaries from Qualcomm AI Hub’s Stable Diffusion v2.1
5+
6+
Please check corresponding section for more information.
7+
8+
## Stable-Diffusion-v2.1
9+
The model architecture, scheduler, and time embedding are from the [stabilityai/stable-diffusion-2-1-base](https://huggingface.co/stabilityai/stable-diffusion-2-1-base).
10+
11+
### Instructions
12+
#### Step 1: Setup
13+
1. Follow the [tutorial](https://pytorch.org/executorch/main/getting-started-setup) to set up ExecuTorch.
14+
2. Follow the [tutorial](https://pytorch.org/executorch/stable/build-run-qualcomm-ai-engine-direct-backend.html) to build Qualcomm AI Engine Direct Backend.
15+
16+
#### Step2: Prepare Model
17+
1. Download the context binaries for TextEncoder, UNet, and VAEDecoder under https://huggingface.co/qualcomm/Stable-Diffusion-v2.1/tree/main
18+
2. Download vocab.json under https://huggingface.co/openai/clip-vit-base-patch32/tree/main
19+
20+
21+
#### Step3: Install Requirements
22+
Before running the code, you need to install the necessary Python packages.
23+
24+
We have verified the code with `diffusers`==0.29.0 and `piq`==0.8.0. Please follow the instructions here to install the required items:
25+
```bash
26+
sh examples/qualcomm/qaihub_scripts/stable_diffusion/install_requirements.sh
27+
```
28+
29+
#### Step4: Run default example
30+
In this example, we execute the script for 20 time steps with the `prompt` 'a photo of an astronaut riding a horse on mars':
31+
```bash
32+
python examples/qualcomm/qaihub_scripts/stable_diffusion/qaihub_stable_diffusion.py -a ${ARTIFACTS} -b build_android -m ${SOC_MODEL} --s ${SERIAL_NUM} --text_encoder_bin ${PATH_TO_TEXT_ENCODER_CONTEXT_BINARY} --unet_bin ${PATH_TO_UNET_CONTEXT_BINARY} --vae_bin ${PATH_TO_VAE_CONTEXT_BINARY} --vocab_json ${PATH_TO_VOCAB_JSON_FILE} --num_time_steps 20 --prompt "a photo of an astronaut riding a horse on mars"
33+
```
34+
- Please replace `${PATH_TO_TEXT_ENCODER_CONTEXT_BINARY}`, `${PATH_TO_UNET_CONTEXT_BINARY}`, and `${PATH_TO_VAE_CONTEXT_BINARY}` with the actual paths to your AI Hub context binary files.
35+
- Please replace `${PATH_TO_VOCAB_JSON_FILE}` with the actual path to your vocab.json file.

‎examples/qualcomm/qaihub_scripts/stable_diffusion/install_requirements.sh

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# For Stable Diffusion V2.1
2+
pip install diffusers==0.29.0
3+
pip install piq==0.8.0

0 commit comments

Comments
 (0)