Skip to content

Vulkan: Add DP4A MMQ and Q8_1 quantization shader #12135

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 13 commits into from
Mar 31, 2025
Merged

Vulkan: Add DP4A MMQ and Q8_1 quantization shader #12135

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
2551791
Vulkan: Add DP4A MMQ and Q8_1 quantization shader
0cc4m Feb 18, 2025
eec67ab
Add q4_0 x q8_1 matrix matrix multiplication support
0cc4m Mar 2, 2025
249595d
Vulkan: Add int8 coopmat MMQ support
0cc4m Mar 10, 2025
34ff5e1
Vulkan: Add q4_1, q5_0 and q5_1 quants, improve integer dot code
0cc4m Mar 21, 2025
2c086fd
Add GL_EXT_integer_dot_product check
0cc4m Mar 22, 2025
45508b4
Remove ggml changes, fix mmq pipeline picker
0cc4m Mar 29, 2025
80a939e
Remove ggml changes, restore Intel coopmat behaviour
0cc4m Mar 29, 2025
e0dedb2
Fix glsl compile attempt when integer vec dot is not supported
0cc4m Mar 29, 2025
a527b9c
Remove redundant code, use non-saturating integer dot, enable all mat…
0cc4m Mar 30, 2025
1da8765
Remove redundant comment
0cc4m Mar 30, 2025
f3dec13
Fix integer dot check
0cc4m Mar 31, 2025
a86c63f
Fix compile issue with unsupported int dot glslc
0cc4m Mar 31, 2025
7f5c84d
Update Windows build Vulkan SDK version
0cc4m Mar 31, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/build.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -803,7 +803,7 @@ jobs:
env:
OPENBLAS_VERSION: 0.3.23
SDE_VERSION: 9.33.0-2024-01-07
VULKAN_VERSION: 1.4.304.1
VULKAN_VERSION: 1.4.309.0

strategy:
matrix:
Expand Down
14 changes: 14 additions & 0 deletions ggml/src/ggml-vulkan/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,20 @@ if (Vulkan_FOUND)
add_compile_definitions(GGML_VULKAN_COOPMAT2_GLSLC_SUPPORT)
endif()

# Compile a test shader to determine whether GL_EXT_integer_dot_product is supported.
# If it's not, there will be an error to stderr.
# If it's supported, set a define to indicate that we should compile those shaders
execute_process(COMMAND ${Vulkan_GLSLC_EXECUTABLE} -o - -fshader-stage=compute --target-env=vulkan1.3 "${CMAKE_CURRENT_SOURCE_DIR}/vulkan-shaders/test_integer_dot_support.comp"
OUTPUT_VARIABLE glslc_output
ERROR_VARIABLE glslc_error)

if (${glslc_error} MATCHES ".*extension not supported: GL_EXT_integer_dot_product.*")
message(STATUS "GL_EXT_integer_dot_product not supported by glslc")
else()
message(STATUS "GL_EXT_integer_dot_product supported by glslc")
add_compile_definitions(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT)
endif()

target_link_libraries(ggml-vulkan PRIVATE Vulkan::Vulkan)
target_include_directories(ggml-vulkan PRIVATE ${CMAKE_CURRENT_BINARY_DIR})

Expand Down
520 changes: 440 additions & 80 deletions ggml/src/ggml-vulkan/ggml-vulkan.cpp
View file
Open in desktop

Large diffs are not rendered by default.

8 changes: 3 additions & 5 deletions ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -212,7 +212,7 @@ void main() {
#else
ACC_TYPE sums[WMITER * TM * WNITER * TN];
FLOAT_TYPE cache_a[WMITER * TM];
FLOAT_TYPE cache_b[WNITER * TN];
FLOAT_TYPE cache_b[TN];

[[unroll]] for (uint i = 0; i < WMITER*TM*WNITER*TN; i++) {
sums[i] = ACC_TYPE(0.0f);
Expand Down Expand Up @@ -744,16 +744,14 @@ void main() {
}
[[unroll]] for (uint wsic = 0; wsic < WNITER; wsic++) {
[[unroll]] for (uint j = 0; j < TN; j++) {
cache_b[wsic * TN + j] = buf_b[(warp_c * WN + wsic * WSUBN + tiwc * TN + j) * SHMEM_STRIDE + i];
cache_b[j] = buf_b[(warp_c * WN + wsic * WSUBN + tiwc * TN + j) * SHMEM_STRIDE + i];
}
}

[[unroll]] for (uint wsic = 0; wsic < WNITER; wsic++) {
[[unroll]] for (uint wsir = 0; wsir < WMITER; wsir++) {
[[unroll]] for (uint cc = 0; cc < TN; cc++) {
[[unroll]] for (uint cr = 0; cr < TM; cr++) {
const uint sums_idx = (wsic * TN + cc) * (WMITER * TM) + wsir * TM + cr;
sums[sums_idx] = fma(ACC_TYPE(cache_a[wsir * TM + cr]), ACC_TYPE(cache_b[wsic * TN + cc]), sums[sums_idx]);
sums[sums_idx] = fma(ACC_TYPE(cache_a[wsir * TM + cr]), ACC_TYPE(cache_b[cc]), sums[sums_idx]);
}
}
}
Expand Down
Loading
Loading