Skip to content

Commit 583c81c

Browse files
airMengairMeng
committed
align GEMM dispatch
1 parent 9335b96 commit 583c81c

File tree

2 files changed

+76
-66
lines changed

2 files changed

+76
-66
lines changed

‎CMakeLists.txt

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -96,8 +96,8 @@ option(LLAMA_LLAMAFILE "llama: use llamafile SGEMM"
9696
set(LLAMA_BLAS_VENDOR "Generic" CACHE STRING "llama: BLAS library vendor")
9797
option(LLAMA_CUDA "llama: use CUDA" OFF)
9898
option(LLAMA_CUBLAS "llama: use CUDA (deprecated, use LLAMA_CUDA)" OFF)
99-
option(LLAMA_CUDA_FORCE_DMMV "llama: use dmmv instead of mmvq CUDA kernels" OFF)
100-
option(LLAMA_CUDA_FORCE_MMQ "llama: use mmq kernels instead of cuBLAS" OFF)
99+
option(LLAMA_FORCE_DMMV "llama: use dmmv instead of mmvq CUDA kernels" OFF)
100+
option(LLAMA_FORCE_MMQ "llama: use mmq kernels instead of cuBLAS" OFF)
101101
set(LLAMA_CUDA_DMMV_X "32" CACHE STRING "llama: x stride for dmmv CUDA kernels")
102102
set(LLAMA_CUDA_MMV_Y "1" CACHE STRING "llama: y block size for mmv CUDA kernels")
103103
option(LLAMA_CUDA_F16 "llama: use 16 bit floats for some calculations" OFF)
@@ -405,10 +405,10 @@ if (LLAMA_CUDA)
405405

406406
add_compile_definitions(GGML_USE_CUDA)
407407
add_compile_definitions(GGML_CUDA_USE_GRAPHS)
408-
if (LLAMA_CUDA_FORCE_DMMV)
408+
if (LLAMA_FORCE_DMMV)
409409
add_compile_definitions(GGML_CUDA_FORCE_DMMV)
410410
endif()
411-
if (LLAMA_CUDA_FORCE_MMQ)
411+
if (LLAMA_FORCE_MMQ)
412412
add_compile_definitions(GGML_CUDA_FORCE_MMQ)
413413
endif()
414414
if (LLAMA_CUDA_NO_VMM)
@@ -578,11 +578,11 @@ if (LLAMA_HIPBLAS)
578578
add_compile_definitions(GGML_HIP_UMA)
579579
endif()
580580

581-
if (LLAMA_CUDA_FORCE_DMMV)
581+
if (LLAMA_FORCE_DMMV)
582582
add_compile_definitions(GGML_CUDA_FORCE_DMMV)
583583
endif()
584584

585-
if (LLAMA_CUDA_FORCE_MMQ)
585+
if (LLAMA_FORCE_MMQ)
586586
add_compile_definitions(GGML_CUDA_FORCE_MMQ)
587587
endif()
588588

@@ -628,6 +628,13 @@ if (LLAMA_SYCL)
628628
add_compile_definitions(GGML_SYCL_F16)
629629
endif()
630630

631+
if (LLAMA_SYCL_FORCE_DMMV)
632+
add_compile_definitions(GGML_SYCL_FORCE_DMMV)
633+
endif()
634+
if (LLAMA_SYCL_FORCE_MMQ)
635+
add_compile_definitions(GGML_SYCL_FORCE_MMQ)
636+
endif()
637+
631638
add_compile_options(-I./) #include DPCT
632639
add_compile_options(-I/${SYCL_INCLUDE_DIR})
633640

‎ggml-sycl.cpp

Lines changed: 63 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -2971,7 +2971,7 @@ static int g_work_group_size = 0;
29712971
// typedef sycl::half ggml_fp16_t;
29722972

29732973
#define __SYCL_ARCH__ DPCT_COMPATIBILITY_TEMP
2974-
#define VER_4VEC 610 //todo for hardward optimize.
2974+
#define VER_4VEC 130 //todo for hardward optimize.
29752975
#define VER_GEN9 700 //todo for hardward optimize.
29762976
#define VER_GEN12 1000000 //todo for hardward optimize.
29772977
#define VER_GEN13 (VER_GEN12 + 1030) //todo for hardward optimize.
@@ -2984,7 +2984,7 @@ static int g_work_group_size = 0;
29842984
#define SYCL_USE_XMX
29852985

29862986
// max batch size to use MMQ kernels when tensor cores are available
2987-
#define XMX_MAX_BATCH_SIZE 32
2987+
#define MMQ_MAX_BATCH_SIZE 32
29882988

29892989

29902990
#if defined(_MSC_VER)
@@ -15193,6 +15193,25 @@ catch (sycl::exception const &exc) {
1519315193
std::exit(1);
1519415194
}
1519515195

15196+
bool ggml_sycl_supports_mmq(enum ggml_type type) {
15197+
// TODO: accuracy issues in MMQ
15198+
return false;
15199+
// switch (type) {
15200+
// case GGML_TYPE_Q4_0:
15201+
// case GGML_TYPE_Q4_1:
15202+
// case GGML_TYPE_Q5_0:
15203+
// case GGML_TYPE_Q5_1:
15204+
// case GGML_TYPE_Q8_0:
15205+
// case GGML_TYPE_Q2_K:
15206+
// case GGML_TYPE_Q3_K:
15207+
// case GGML_TYPE_Q4_K:
15208+
// case GGML_TYPE_Q5_K:
15209+
// case GGML_TYPE_Q6_K:
15210+
// return true;
15211+
// default:
15212+
// return false;
15213+
// }
15214+
}
1519615215

1519715216
static void ggml_sycl_mul_mat(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
1519815217
const bool all_on_device =
@@ -15209,76 +15228,60 @@ static void ggml_sycl_mul_mat(const ggml_tensor * src0, const ggml_tensor * src1
1520915228
}
1521015229
}
1521115230

15231+
#if !defined(GGML_SYCL_FORCE_MMQ)
15232+
#define SYCL_USE_XMX
15233+
#endif
15234+
1521215235
#ifdef SYCL_USE_XMX
15213-
const bool use_xmx = true;
15236+
bool use_xmx = true;
1521415237
#else
15215-
const bool use_xmx = false;
15238+
bool use_xmx = false;
1521615239
#endif
1521715240

15218-
// debug helpers
15219-
//printf("src0: %8d %8d %8d %8d\n", src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3]);
15220-
//printf(" %8d %8d %8d %8d\n", src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3]);
15221-
//printf("src1: %8d %8d %8d %8d\n", src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3]);
15222-
//printf(" %8d %8d %8d %8d\n", src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3]);
15223-
//printf("src0 is contiguous %d, transposed %d, type = %s, name = %s\n", ggml_is_contiguous(src0), ggml_is_transposed(src0), ggml_type_name(src0->type), src0->name);
15224-
//printf("src1 is contiguous %d, transposed %d, type = %s, name = %s\n", ggml_is_contiguous(src1), ggml_is_transposed(src1), ggml_type_name(src1->type), src1->name);
15241+
// check data types and tensor shapes for custom matrix multiplication kernels:
15242+
bool use_dequantize_mul_mat_vec = (ggml_is_quantized(src0->type) || src0->type == GGML_TYPE_F16)
15243+
&& src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32
15244+
&& src0->ne[0] % GGML_SYCL_DMMV_X == 0 && src1->ne[1] == 1;
15245+
15246+
bool use_mul_mat_vec_q = ggml_is_quantized(src0->type)
15247+
&& src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32
15248+
&& src1->ne[1] <= MMVQ_MAX_BATCH_SIZE;
15249+
15250+
bool use_mul_mat_q = ggml_sycl_supports_mmq(src0->type)
15251+
&& src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32;
15252+
15253+
// fp16 performance always better on gen12+
15254+
const bool fp16_performance_good = true;
1522515255

15226-
if (!split && all_on_device && !use_xmx && src0->type == GGML_TYPE_F16 && ggml_is_permuted(src0) && ggml_is_permuted(src1) && src1->ne[1] == 1) {
15256+
// mmvq and mmq need the __dp4a instruction which is available for gen12+
15257+
use_mul_mat_vec_q = use_mul_mat_vec_q; // Check dp4a
15258+
// Workaround in https://github.com/ggerganov/llama.cpp/commit/95f84d5ce8b449a9b16009434aca800df504a02e
15259+
use_mul_mat_q = use_mul_mat_q && (src0->type != GGML_TYPE_IQ2_XXS);
15260+
#ifdef SYCL_USE_XMX
15261+
use_mul_mat_q = use_mul_mat_q && (!fp16_performance_good || src1->ne[1] <= MMQ_MAX_BATCH_SIZE);
15262+
#endif // SYCL_USE_XMX
15263+
15264+
#infdef GGML_SYCL_FORCE_DMMV
15265+
use_dequantize_mul_mat_vec = use_dequantize_mul_mat_vec && !use_mul_mat_vec_q;
15266+
#endif // GGML_SYCL_FORCE_DMMV
15267+
15268+
if (!split && !fp16_performance_good && src0->type == GGML_TYPE_F16 && ggml_is_permuted(src0) && ggml_is_permuted(src1) && src1->ne[1] == 1) {
1522715269
// KQ single-batch
15228-
// GGML_SYCL_DEBUG("ggml_sycl_mul_mat_vec_p021\n");
1522915270
ggml_sycl_mul_mat_vec_p021(src0, src1, dst);
15230-
} else if (!split && all_on_device && !use_xmx && src0->type == GGML_TYPE_F16 && !ggml_is_contiguous(src0) && !ggml_is_transposed(src1) && src1->ne[1] == 1) {
15271+
} else if (!split && !fp16_performance_good && src0->type == GGML_TYPE_F16 && !ggml_is_contiguous(src0) && !ggml_is_transposed(src1) && src1->ne[1] == 1) {
1523115272
// KQV single-batch
15232-
// GGML_SYCL_DEBUG("ggml_sycl_mul_mat_vec_nc\n");
1523315273
ggml_sycl_mul_mat_vec_nc(src0, src1, dst);
15234-
} else if (!split && all_on_device && use_xmx && src0->type == GGML_TYPE_F16 && !ggml_is_transposed(src0) && !ggml_is_transposed(src1)) {
15274+
} else if (!split && src0->type == GGML_TYPE_F16 && (src1->type == GGML_TYPE_F16 || fp16_performance_good) && !ggml_is_transposed(src0) && !ggml_is_transposed(src1) && src1->ne[2]*src1->ne[3] > 1) {
1523515275
// KQ + KQV multi-batch
15236-
// GGML_SYCL_DEBUG("ggml_sycl_mul_mat_batched_sycl\n");
1523715276
ggml_sycl_mul_mat_batched_sycl(src0, src1, dst);
15238-
} else if (src0->type == GGML_TYPE_F32) {
15239-
// GGML_SYCL_DEBUG("ggml_sycl_op_mul_mat\n");
15240-
ggml_sycl_op_mul_mat(src0, src1, dst, ggml_sycl_op_mul_mat_sycl, false);
15241-
} else if (ggml_is_quantized(src0->type) || src0->type == GGML_TYPE_F16) {
15242-
// GGML_SYCL_DEBUG("ggml_is_quantized or GGML_TYPE_F16\n");
15243-
if (src1->ne[1] == 1 && src0->ne[0] % GGML_SYCL_DMMV_X == 0) {
15244-
#ifdef GGML_SYCL_FORCE_DMMV
15245-
const bool use_mul_mat_vec_q = false;
15246-
#else
15247-
bool use_mul_mat_vec_q = min_compute_capability >= VER_4VEC && ggml_is_quantized(src0->type);
15248-
use_mul_mat_vec_q = use_mul_mat_vec_q ||
15249-
(src0->type == GGML_TYPE_IQ2_XXS) || (src0->type == GGML_TYPE_IQ2_XS) || (src0->type == GGML_TYPE_IQ2_S) ||
15250-
(src0->type == GGML_TYPE_IQ3_XXS) || (src0->type == GGML_TYPE_IQ3_S) ||
15251-
(src0->type == GGML_TYPE_IQ4_NL) || (src0->type == GGML_TYPE_IQ4_XS) ||
15252-
(src0->type == GGML_TYPE_IQ1_S) || (src0->type == GGML_TYPE_IQ1_M);
15253-
15254-
15255-
#endif // GGML_SYCL_FORCE_DMMV
15256-
15257-
if (use_mul_mat_vec_q) {
15258-
// GGML_SYCL_DEBUG("ggml_sycl_mul_mat ggml_sycl_op_mul_mat_vec_q path\n");
15259-
ggml_sycl_op_mul_mat(src0, src1, dst, ggml_sycl_op_mul_mat_vec_q, true);
15260-
} else {
15261-
// GGML_SYCL_DEBUG("ggml_sycl_mul_mat ggml_sycl_op_dequantize_mul_mat_vec path\n");
15262-
ggml_sycl_op_mul_mat(src0, src1, dst, ggml_sycl_op_dequantize_mul_mat_vec, false);
15263-
}
15264-
} else {
15265-
bool use_mul_mat_q = min_compute_capability >= VER_4VEC && ggml_is_quantized(src0->type);
15266-
use_mul_mat_q = use_mul_mat_q && (src0->type != GGML_TYPE_IQ2_XXS);
15267-
15268-
if (use_xmx && min_compute_capability >= VER_GEN9 && src1->ne[1] > XMX_MAX_BATCH_SIZE) {
15269-
use_mul_mat_q = false;
15270-
}
15271-
15272-
if (use_mul_mat_q) {
15273-
// GGML_SYCL_DEBUG("ggml_sycl_mul_mat ggml_sycl_op_mul_mat_q path\n");
15274-
ggml_sycl_op_mul_mat(src0, src1, dst, ggml_sycl_op_mul_mat_q, true);
15275-
} else {
15276-
// GGML_SYCL_DEBUG("ggml_sycl_mul_mat ggml_sycl_op_mul_mat_sycl path\n");
15277-
ggml_sycl_op_mul_mat(src0, src1, dst, ggml_sycl_op_mul_mat_sycl, false);
15278-
}
15279-
}
15277+
} else if (use_dequantize_mul_mat_vec) {
15278+
ggml_sycl_op_mul_mat(src0, src1, dst, ggml_sycl_op_dequantize_mul_mat_vec, false);
15279+
} else if (use_mul_mat_vec_q) {
15280+
ggml_sycl_op_mul_mat(src0, src1, dst, ggml_sycl_op_mul_mat_vec_q, true);
15281+
} else if (use_mul_mat_q) {
15282+
ggml_sycl_op_mul_mat(src0, src1, dst, ggml_sycl_op_mul_mat_q, true);
1528015283
} else {
15281-
GGML_ASSERT(false);
15284+
ggml_sycl_op_mul_mat(src0, src1, dst, ggml_sycl_op_mul_mat_sycl, false);
1528215285
}
1528315286
}
1528415287

0 commit comments

Comments
 (0)