Skip to content

Commit bc63230

Browse files
SS-JIAfacebook-github-bot
authored andcommitted
Parallelize SPIR-V compilation (#4200)
Summary: Pull Request resolved: #4200 Due to a high number of shaders to compile, shader compilation has become quite slow (taking around 1 minute from empirical observations). This change updates the SPIR-V compilation script to parallelize SPIR-V compilation over the available number of CPU cores. Testing the impact of parallelization using the following command: ``` buck build //xplat/executorch/backends/vulkan:gen_vulkan_graph_runtime_shaderlib_cpp --out ~/scratch/shaders ``` We can observe the following improvement: | | Before | After| |--- | --- | --- | | M1 Mac | 42.4s | 7.8s | Reviewed By: liuk22, jorgep31415 Differential Revision: D59597186 fbshipit-source-id: 72df756a3a1f818688af66bbce4fc9fc02f1f505
1 parent f80c6ef commit bc63230

File tree

1 file changed

+14
-4
lines changed

1 file changed

+14
-4
lines changed

backends/vulkan/runtime/gen_vulkan_spv.py

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import re
1616
import sys
1717
from itertools import product
18+
from multiprocessing.pool import ThreadPool
1819

1920
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
2021
import subprocess
@@ -620,9 +621,12 @@ def constructOutputMap(self) -> None:
620621

621622
def generateSPV(self, output_dir: str) -> Dict[str, str]:
622623
output_file_map = {}
623-
for shader_name in self.output_shader_map:
624-
source_glsl = self.output_shader_map[shader_name][0]
625-
shader_params = self.output_shader_map[shader_name][1]
624+
625+
def process_shader(shader_paths_pair):
626+
shader_name = shader_paths_pair[0]
627+
628+
source_glsl = shader_paths_pair[1][0]
629+
shader_params = shader_paths_pair[1][1]
626630

627631
with codecs.open(source_glsl, "r", encoding="utf-8") as input_file:
628632
input_text = input_file.read()
@@ -652,9 +656,15 @@ def generateSPV(self, output_dir: str) -> Dict[str, str]:
652656
]
653657

654658
print("glslc cmd:", cmd)
655-
# pyre-ignore
656659
subprocess.check_call(cmd)
657660

661+
return (spv_out_path, glsl_out_path)
662+
663+
# Parallelize shader compilation as much as possible to optimize build time.
664+
with ThreadPool(os.cpu_count()) as pool:
665+
for spv_out_path, glsl_out_path in pool.map(
666+
process_shader, self.output_shader_map.items()
667+
):
658668
output_file_map[spv_out_path] = glsl_out_path
659669

660670
return output_file_map

0 commit comments

Comments
 (0)