@@ -103,6 +103,7 @@ option(LLAMA_VULKAN "llama: use Vulkan"
103103option (LLAMA_METAL "llama: use Metal" ${LLAMA_METAL_DEFAULT} )
104104option (LLAMA_METAL_NDEBUG "llama: disable Metal debugging" OFF )
105105option (LLAMA_METAL_SHADER_DEBUG "llama: compile Metal with -fno-fast-math" OFF )
106+ option (LLAMA_KOMPUTE "llama: use Kompute" OFF )
106107option (LLAMA_MPI "llama: use MPI" OFF )
107108option (LLAMA_QKK_64 "llama: use super-block size of 64 for k-quants" OFF )
108109option (LLAMA_SYCL "llama: use SYCL" OFF )
@@ -422,7 +423,13 @@ if (LLAMA_VULKAN)
422423 if (Vulkan_FOUND )
423424 message (STATUS "Vulkan found" )
424425
426+ set (GGML_HEADERS_VULKAN ggml-vulkan.h )
427+ set (GGML_SOURCES_VULKAN ggml-vulkan.cpp )
428+
425429 add_library (ggml-vulkan STATIC ggml-vulkan.cpp ggml-vulkan.h )
430+ if (BUILD_SHARED_LIBS )
431+ set_target_properties (ggml-vulkan PROPERTIES POSITION_INDEPENDENT_CODE ON )
432+ endif ()
426433 target_link_libraries (ggml-vulkan PRIVATE Vulkan::Vulkan )
427434
428435 add_compile_definitions (GGML_USE_VULKAN )
@@ -478,7 +485,6 @@ if (LLAMA_HIPBLAS)
478485 endif ()
479486endif ()
480487
481-
482488if (LLAMA_SYCL )
483489 if ( NOT DEFINED ENV{ONEAPI_ROOT} )
484490 message (FATAL_ERROR "Not detect ENV {ONEAPI_ROOT}, please install oneAPI & source it, like: source /opt/intel/oneapi/setvars.sh" )
@@ -501,7 +507,165 @@ if (LLAMA_SYCL)
501507 set (GGML_HEADERS_SYCL ggml.h ggml-sycl.h )
502508 set (GGML_SOURCES_SYCL ggml-sycl.cpp )
503509
504- set (LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} sycl OpenCL mkl_core pthread m dl mkl_sycl_blas mkl_intel_ilp64 mkl_tbb_thread )
510+ if (WIN32 )
511+ set (LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} -fsycl sycl7 OpenCL mkl_sycl_blas_dll.lib mkl_intel_ilp64_dll.lib mkl_sequential_dll.lib mkl_core_dll.lib )
512+ else ()
513+ set (LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} -fsycl OpenCL mkl_core pthread m dl mkl_sycl_blas mkl_intel_ilp64 mkl_tbb_thread )
514+ endif ()
515+ endif ()
516+
517+ if (LLAMA_KOMPUTE )
518+ add_compile_definitions (VULKAN_HPP_DISPATCH_LOADER_DYNAMIC=1 )
519+ find_package (Vulkan COMPONENTS glslc REQUIRED )
520+ find_program (glslc_executable NAMES glslc HINTS Vulkan::glslc )
521+ if (NOT glslc_executable )
522+ message (FATAL_ERROR "glslc not found" )
523+ endif ()
524+
525+ function (compile_shader )
526+ set (options )
527+ set (oneValueArgs )
528+ set (multiValueArgs SOURCES )
529+ cmake_parse_arguments (compile_shader "${options} " "${oneValueArgs} " "${multiValueArgs} " ${ARGN} )
530+ foreach (source ${compile_shader_SOURCES} )
531+ get_filename_component (filename ${source} NAME )
532+ set (spv_file ${filename} .spv )
533+ add_custom_command (
534+ OUTPUT ${spv_file}
535+ DEPENDS ${CMAKE_CURRENT_SOURCE_DIR} /${source}
536+ ${CMAKE_CURRENT_SOURCE_DIR} /kompute-shaders/common.comp
537+ ${CMAKE_CURRENT_SOURCE_DIR} /kompute-shaders/op_getrows.comp
538+ ${CMAKE_CURRENT_SOURCE_DIR} /kompute-shaders/op_mul_mv_q_n_pre.comp
539+ ${CMAKE_CURRENT_SOURCE_DIR} /kompute-shaders/op_mul_mv_q_n.comp
540+ COMMAND ${glslc_executable} --target-env=vulkan1.2 -o ${spv_file} ${CMAKE_CURRENT_SOURCE_DIR} /${source}
541+ COMMENT "Compiling ${source} to ${spv_file} "
542+ )
543+
544+ get_filename_component (RAW_FILE_NAME ${spv_file} NAME )
545+ set (FILE_NAME "shader${RAW_FILE_NAME} " )
546+ string (REPLACE ".comp.spv" ".h" HEADER_FILE ${FILE_NAME} )
547+ string (TOUPPER ${HEADER_FILE} HEADER_FILE_DEFINE )
548+ string (REPLACE "." "_" HEADER_FILE_DEFINE "${HEADER_FILE_DEFINE} " )
549+ set (OUTPUT_HEADER_FILE "${HEADER_FILE} " )
550+ message (STATUS "${HEADER_FILE} generating ${HEADER_FILE_DEFINE} " )
551+ if (CMAKE_GENERATOR MATCHES "Visual Studio" )
552+ add_custom_command (
553+ OUTPUT ${OUTPUT_HEADER_FILE}
554+ COMMAND ${CMAKE_COMMAND} -E echo "/*THIS FILE HAS BEEN AUTOMATICALLY GENERATED - DO NOT EDIT*/" > ${OUTPUT_HEADER_FILE}
555+ COMMAND ${CMAKE_COMMAND} -E echo \"\# ifndef ${HEADER_FILE_DEFINE} \" >> ${OUTPUT_HEADER_FILE}
556+ COMMAND ${CMAKE_COMMAND} -E echo \"\# define ${HEADER_FILE_DEFINE} \" >> ${OUTPUT_HEADER_FILE}
557+ COMMAND ${CMAKE_COMMAND} -E echo " namespace kp { " >> ${OUTPUT_HEADER_FILE}
558+ COMMAND ${CMAKE_COMMAND} -E echo " namespace shader_data { " >> ${OUTPUT_HEADER_FILE}
559+ COMMAND ${CMAKE_BINARY_DIR} /bin/$<CONFIG>/xxd -i ${RAW_FILE_NAME} >> ${OUTPUT_HEADER_FILE}
560+ COMMAND ${CMAKE_COMMAND} -E echo " }} " >> ${OUTPUT_HEADER_FILE}
561+ COMMAND ${CMAKE_COMMAND} -E echo \"\# endif // define ${HEADER_FILE_DEFINE} \" >> ${OUTPUT_HEADER_FILE}
562+ DEPENDS ${spv_file} xxd
563+ COMMENT " Converting to hpp: ${FILE_NAME} ${CMAKE_BINARY_DIR} /bin/$<CONFIG>/xxd "
564+ )
565+ else()
566+ add_custom_command(
567+ OUTPUT ${OUTPUT_HEADER_FILE}
568+ COMMAND ${CMAKE_COMMAND} -E echo " /*THIS FILE HAS BEEN AUTOMATICALLY GENERATED - DO NOT EDIT*/ " > ${OUTPUT_HEADER_FILE}
569+ COMMAND ${CMAKE_COMMAND} -E echo \"\# ifndef ${HEADER_FILE_DEFINE} \" >> ${OUTPUT_HEADER_FILE}
570+ COMMAND ${CMAKE_COMMAND} -E echo \"\# define ${HEADER_FILE_DEFINE} \" >> ${OUTPUT_HEADER_FILE}
571+ COMMAND ${CMAKE_COMMAND} -E echo " namespace kp { " >> ${OUTPUT_HEADER_FILE}
572+ COMMAND ${CMAKE_COMMAND} -E echo " namespace shader_data { " >> ${OUTPUT_HEADER_FILE}
573+ COMMAND ${CMAKE_BINARY_DIR} /bin/xxd -i ${RAW_FILE_NAME} >> ${OUTPUT_HEADER_FILE}
574+ COMMAND ${CMAKE_COMMAND} -E echo " }} " >> ${OUTPUT_HEADER_FILE}
575+ COMMAND ${CMAKE_COMMAND} -E echo \"\# endif // define ${HEADER_FILE_DEFINE} \" >> ${OUTPUT_HEADER_FILE}
576+ DEPENDS ${spv_file} xxd
577+ COMMENT " Converting to hpp: ${FILE_NAME} ${CMAKE_BINARY_DIR} /bin/xxd "
578+ )
579+ endif()
580+ endforeach()
581+ endfunction()
582+
583+ if (EXISTS " ${CMAKE_CURRENT_SOURCE_DIR} /kompute/CMakeLists.txt ")
584+ message(STATUS " Kompute found ")
585+ set(KOMPUTE_OPT_LOG_LEVEL Error CACHE STRING " Kompute log level ")
586+ add_subdirectory(kompute)
587+
588+ # Compile our shaders
589+ compile_shader(SOURCES
590+ kompute-shaders/op_scale.comp
591+ kompute-shaders/op_scale_8.comp
592+ kompute-shaders/op_add.comp
593+ kompute-shaders/op_addrow.comp
594+ kompute-shaders/op_mul.comp
595+ kompute-shaders/op_silu.comp
596+ kompute-shaders/op_relu.comp
597+ kompute-shaders/op_gelu.comp
598+ kompute-shaders/op_softmax.comp
599+ kompute-shaders/op_norm.comp
600+ kompute-shaders/op_rmsnorm.comp
601+ kompute-shaders/op_diagmask.comp
602+ kompute-shaders/op_mul_mat_mat_f32.comp
603+ kompute-shaders/op_mul_mat_f16.comp
604+ kompute-shaders/op_mul_mat_q8_0.comp
605+ kompute-shaders/op_mul_mat_q4_0.comp
606+ kompute-shaders/op_mul_mat_q4_1.comp
607+ kompute-shaders/op_mul_mat_q6_k.comp
608+ kompute-shaders/op_getrows_f16.comp
609+ kompute-shaders/op_getrows_q4_0.comp
610+ kompute-shaders/op_getrows_q4_1.comp
611+ kompute-shaders/op_getrows_q6_k.comp
612+ kompute-shaders/op_rope_f16.comp
613+ kompute-shaders/op_rope_f32.comp
614+ kompute-shaders/op_cpy_f16_f16.comp
615+ kompute-shaders/op_cpy_f16_f32.comp
616+ kompute-shaders/op_cpy_f32_f16.comp
617+ kompute-shaders/op_cpy_f32_f32.comp
618+ )
619+
620+ # Create a custom target for our generated shaders
621+ add_custom_target(generated_shaders DEPENDS
622+ shaderop_scale.h
623+ shaderop_scale_8.h
624+ shaderop_add.h
625+ shaderop_addrow.h
626+ shaderop_mul.h
627+ shaderop_silu.h
628+ shaderop_relu.h
629+ shaderop_gelu.h
630+ shaderop_softmax.h
631+ shaderop_norm.h
632+ shaderop_rmsnorm.h
633+ shaderop_diagmask.h
634+ shaderop_mul_mat_mat_f32.h
635+ shaderop_mul_mat_f16.h
636+ shaderop_mul_mat_q8_0.h
637+ shaderop_mul_mat_q4_0.h
638+ shaderop_mul_mat_q4_1.h
639+ shaderop_mul_mat_q6_k.h
640+ shaderop_getrows_f16.h
641+ shaderop_getrows_q4_0.h
642+ shaderop_getrows_q4_1.h
643+ shaderop_getrows_q6_k.h
644+ shaderop_rope_f16.h
645+ shaderop_rope_f32.h
646+ shaderop_cpy_f16_f16.h
647+ shaderop_cpy_f16_f32.h
648+ shaderop_cpy_f32_f16.h
649+ shaderop_cpy_f32_f32.h
650+ )
651+
652+ # Create a custom command that depends on the generated_shaders
653+ add_custom_command(
654+ OUTPUT ${CMAKE_CURRENT_BINARY_DIR} /ggml-kompute.stamp
655+ COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR} /ggml-kompute.stamp
656+ DEPENDS generated_shaders
657+ COMMENT " Ensuring shaders are generated before compiling ggml-kompute.cpp "
658+ )
659+
660+ # Add the stamp to the main sources to ensure dependency tracking
661+ set(GGML_SOURCES_KOMPUTE ggml-kompute.cpp ${CMAKE_CURRENT_BINARY_DIR} /ggml-kompute.stamp)
662+ set(GGML_HEADERS_KOMPUTE ggml-kompute.h ${CMAKE_CURRENT_BINARY_DIR} /ggml-kompute.stamp)
663+ add_compile_definitions(GGML_USE_KOMPUTE)
664+ set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} kompute)
665+ set(LLAMA_EXTRA_INCLUDES ${LLAMA_EXTRA_INCLUDES} ${CMAKE_BINARY_DIR} )
666+ else()
667+ message(WARNING " Kompute not found ")
668+ endif()
505669endif()
506670
507671function(get_flags CCID CCVER)
@@ -846,12 +1010,14 @@ add_library(ggml OBJECT
8461010 ggml-backend.h
8471011 ggml-quants.c
8481012 ggml-quants.h
849- ${GGML_SOURCES_CUDA} ${GGML_HEADERS_CUDA}
850- ${GGML_SOURCES_OPENCL} ${GGML_HEADERS_OPENCL}
851- ${GGML_SOURCES_METAL} ${GGML_HEADERS_METAL}
852- ${GGML_SOURCES_MPI} ${GGML_HEADERS_MPI}
853- ${GGML_SOURCES_EXTRA} ${GGML_HEADERS_EXTRA}
854- ${GGML_SOURCES_SYCL} ${GGML_HEADERS_SYCL}
1013+ ${GGML_SOURCES_CUDA} ${GGML_HEADERS_CUDA}
1014+ ${GGML_SOURCES_OPENCL} ${GGML_HEADERS_OPENCL}
1015+ ${GGML_SOURCES_VULKAN} ${GGML_HEADERS_VULKAN}
1016+ ${GGML_SOURCES_METAL} ${GGML_HEADERS_METAL}
1017+ ${GGML_SOURCES_MPI} ${GGML_HEADERS_MPI}
1018+ ${GGML_SOURCES_EXTRA} ${GGML_HEADERS_EXTRA}
1019+ ${GGML_SOURCES_SYCL} ${GGML_HEADERS_SYCL}
1020+ ${GGML_SOURCES_KOMPUTE} ${GGML_HEADERS_KOMPUTE}
8551021 )
8561022
8571023target_include_directories(ggml PUBLIC . ${LLAMA_EXTRA_INCLUDES} )
@@ -928,7 +1094,7 @@ install(FILES ${CMAKE_CURRENT_BINARY_DIR}/LlamaConfig.cmake
9281094 DESTINATION ${CMAKE_INSTALL_LIBDIR} /cmake/Llama)
9291095
9301096set(GGML_PUBLIC_HEADERS " ggml.h " " ggml-alloc.h " " ggml-backend.h "
931- "${GGML_HEADERS_CUDA} " "${GGML_HEADERS_OPENCL} "
1097+ " ${GGML_HEADERS_CUDA} " " ${GGML_HEADERS_OPENCL} " " ${GGML_HEADERS_VULKAN} "
9321098 " ${GGML_HEADERS_METAL} " " ${GGML_HEADERS_MPI} " " ${GGML_HEADERS_EXTRA} ")
9331099
9341100set_target_properties(ggml PROPERTIES PUBLIC_HEADER " ${GGML_PUBLIC_HEADERS} ")
0 commit comments