Skip to content

Commit 4dc5515

Browse files
luoyu-intelluoyu-intel
committed
add dnnl stream
1 parent 3d0a64f commit 4dc5515

File tree

3 files changed

+69
-35
lines changed

3 files changed

+69
-35
lines changed

‎ggml/src/ggml-sycl.cpp

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2494,7 +2494,8 @@ inline void ggml_sycl_op_mul_mat_sycl(
24942494
const to_fp32_sycl_t to_fp32_sycl = ggml_get_to_fp32_sycl(GGML_TYPE_F16);
24952495
to_fp32_sycl(dst_f16.get(), dst_dd_i, row_diff*src1_ncols, stream);
24962496
#else
2497-
DnnlGemmWrapper::row_gemm(*stream, false, true, src1_ncols, row_diff, ne10, src1_ptr, DnnlGemmWrapper::to_dt<sycl::half>(),
2497+
auto dnnl_stream = ctx.stream_dnnl(stream);
2498+
DnnlGemmWrapper::row_gemm(dnnl_stream, false, true, src1_ncols, row_diff, ne10, src1_ptr, DnnlGemmWrapper::to_dt<sycl::half>(),
24982499
src0_ptr, DnnlGemmWrapper::to_dt<sycl::half>(), dst_f16.get(), DnnlGemmWrapper::to_dt<sycl::half>());
24992500
const to_fp32_sycl_t to_fp32_sycl = ggml_get_to_fp32_sycl(GGML_TYPE_F16);
25002501
to_fp32_sycl(dst_f16.get(), dst_dd_i, row_diff* src1_ncols, stream);
@@ -2529,7 +2530,8 @@ inline void ggml_sycl_op_mul_mat_sycl(
25292530
src1_ddf1_i, ne10, dpct::get_value(&beta, *stream),
25302531
dst_dd_i, ldc)));
25312532
#else
2532-
DnnlGemmWrapper::row_gemm(*stream, false, true, src1_ncols, row_diff, ne10, src1_ddf1_i, DnnlGemmWrapper::to_dt<float>(),
2533+
auto dnnl_stream = ctx.stream_dnnl(stream);
2534+
DnnlGemmWrapper::row_gemm(dnnl_stream, false, true, src1_ncols, row_diff, ne10, src1_ddf1_i, DnnlGemmWrapper::to_dt<float>(),
25332535
src0_ddf_i, DnnlGemmWrapper::to_dt<float>(), dst_dd_i, DnnlGemmWrapper::to_dt<float>());
25342536
#endif
25352537
}

‎ggml/src/ggml-sycl/common.hpp

Lines changed: 62 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,10 @@
1919
#include "dpct/helper.hpp"
2020
#include "ggml-sycl.h"
2121
#include "presets.hpp"
22+
#if GGML_SYCL_DNNL
23+
#include "dnnl.hpp"
24+
#include "dnnl_sycl.hpp"
25+
#endif
2226

2327
#define GGML_COMMON_DECL_SYCL
2428
#define GGML_COMMON_IMPL_SYCL
@@ -59,7 +63,7 @@ static int g_ggml_sycl_debug = 0;
5963
// define for XMX in Intel GPU
6064
// TODO: currently, it's not used for XMX really.
6165
#if !defined(GGML_SYCL_FORCE_MMQ)
62-
#define SYCL_USE_XMX
66+
#define SYCL_USE_XMX
6367
#endif
6468

6569
// max batch size to use MMQ kernels when tensor cores are available
@@ -80,16 +84,16 @@ static int g_ggml_sycl_debug = 0;
8084
typedef sycl::queue *queue_ptr;
8185

8286
enum ggml_sycl_backend_gpu_mode {
83-
SYCL_UNSET_GPU_MODE = -1,
84-
SYCL_SINGLE_GPU_MODE = 0,
85-
SYCL_MUL_GPU_MODE
87+
SYCL_UNSET_GPU_MODE = -1,
88+
SYCL_SINGLE_GPU_MODE = 0,
89+
SYCL_MUL_GPU_MODE
8690
};
8791

8892
static_assert(sizeof(sycl::half) == sizeof(ggml_fp16_t), "wrong fp16 size");
8993

9094
static void crash() {
91-
int* ptr = NULL;
92-
*ptr = 0;
95+
int* ptr = NULL;
96+
*ptr = 0;
9397
}
9498

9599
[[noreturn]] static void ggml_sycl_error(
@@ -98,9 +102,9 @@ static void crash() {
98102
const char* file,
99103
const int line,
100104
const char* msg) {
101-
fprintf(stderr, "SYCL error: %s: %s\n", stmt, msg);
102-
fprintf(stderr, " in function %s at %s:%d\n", func, file, line);
103-
GGML_ABORT("SYCL error");
105+
fprintf(stderr, "SYCL error: %s: %s\n", stmt, msg);
106+
fprintf(stderr, " in function %s at %s:%d\n", func, file, line);
107+
GGML_ABORT("SYCL error");
104108
}
105109

106110
#define SYCL_CHECK(err) \
@@ -138,40 +142,40 @@ static int g_all_sycl_device_count = -1;
138142
static bool g_ggml_backend_sycl_buffer_type_initialized = false;
139143

140144
static ggml_sycl_backend_gpu_mode g_ggml_sycl_backend_gpu_mode =
141-
SYCL_UNSET_GPU_MODE;
145+
SYCL_UNSET_GPU_MODE;
142146

143147
static void* g_scratch_buffer = nullptr;
144148
static size_t g_scratch_size = 0; // disabled by default
145149
static size_t g_scratch_offset = 0;
146150

147151
[[noreturn]] static inline void bad_arch(const sycl::stream& stream_ct1) {
148-
stream_ct1 << "ERROR: ggml-sycl was compiled without support for the "
149-
"current GPU architecture.\n";
150-
// __trap();
151-
std::exit(1);
152+
stream_ct1 << "ERROR: ggml-sycl was compiled without support for the "
153+
"current GPU architecture.\n";
154+
// __trap();
155+
std::exit(1);
152156

153-
(void)bad_arch; // suppress unused function warning
157+
(void)bad_arch; // suppress unused function warning
154158
}
155159

156160
int get_current_device_id();
157161

158162
inline dpct::err0 ggml_sycl_set_device(const int device) try {
159163

160-
int current_device_id;
161-
SYCL_CHECK(CHECK_TRY_ERROR(current_device_id = get_current_device_id()));
164+
int current_device_id;
165+
SYCL_CHECK(CHECK_TRY_ERROR(current_device_id = get_current_device_id()));
162166

163-
// GGML_SYCL_DEBUG("ggml_sycl_set_device device_id=%d,
164-
// current_device_id=%d\n", device, current_device);
165-
if (device == current_device_id) {
166-
return 0;
167-
}
167+
// GGML_SYCL_DEBUG("ggml_sycl_set_device device_id=%d,
168+
// current_device_id=%d\n", device, current_device);
169+
if (device == current_device_id) {
170+
return 0;
171+
}
168172

169-
return CHECK_TRY_ERROR(dpct::select_device(device));
173+
return CHECK_TRY_ERROR(dpct::select_device(device));
170174
} catch (sycl::exception const& exc) {
171-
std::cerr << exc.what() << "Exception caught at file:" << __FILE__
172-
<< ", line:" << __LINE__ << std::endl;
173-
crash();
174-
std::exit(1);
175+
std::cerr << exc.what() << "Exception caught at file:" << __FILE__
176+
<< ", line:" << __LINE__ << std::endl;
177+
crash();
178+
std::exit(1);
175179
}
176180

177181
//////////////////////
@@ -249,10 +253,10 @@ struct ggml_sycl_pool_alloc {
249253
// backend interface
250254

251255
struct ggml_tensor_extra_gpu {
252-
void* data_device[GGML_SYCL_MAX_DEVICES]; // 1 pointer for each device for split
253-
// tensors
254-
dpct::event_ptr events[GGML_SYCL_MAX_DEVICES]
255-
[GGML_SYCL_MAX_STREAMS]; // events for synchronizing multiple GPUs
256+
void* data_device[GGML_SYCL_MAX_DEVICES]; // 1 pointer for each device for split
257+
// tensors
258+
dpct::event_ptr events[GGML_SYCL_MAX_DEVICES]
259+
[GGML_SYCL_MAX_STREAMS]; // events for synchronizing multiple GPUs
256260
};
257261

258262
struct ggml_backend_sycl_context {
@@ -277,6 +281,33 @@ struct ggml_backend_sycl_context {
277281
return stream(device, 0);
278282
}
279283

284+
#if GGML_SYCL_DNNL
285+
dnnl::stream make_stream(sycl::queue& q) {
286+
// Get the device associated with the queue
287+
sycl::device dev = q.get_device();
288+
// Get the context associated with the queue
289+
sycl::context ctx = q.get_context();
290+
const dnnl::engine eng = dnnl::sycl_interop::make_engine(dev, ctx);
291+
dnnl::stream stream = dnnl::sycl_interop::make_stream(eng, q);
292+
return stream;
293+
}
294+
std::unordered_map<sycl::queue*, dnnl::stream> stream_map;
295+
dnnl::stream stream_dnnl(int device, int _stream) {
296+
auto q = stream(device, _stream);
297+
return stream_dnnl(q);
298+
}
299+
dnnl::stream stream_dnnl(sycl::queue* qptr) {
300+
auto it = stream_map.find(qptr);
301+
if (it == stream_map.end()) {
302+
stream_map[qptr] = make_stream(*qptr);
303+
}
304+
return it->second;
305+
}
306+
dnnl::stream stream_dnnl() {
307+
return stream_dnnl(device, 0);
308+
}
309+
#endif
310+
280311
// pool
281312
std::unique_ptr<ggml_sycl_pool> pools[GGML_SYCL_MAX_DEVICES];
282313

‎ggml/src/ggml-sycl/gemm.hpp

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,12 @@
1717
#include <iostream>
1818

1919
#include "ggml-sycl.h"
20-
#include "dnnl.hpp"
21-
#include "dnnl_sycl.hpp"
2220

2321
#if GGML_SYCL_DNNL
2422

23+
#include "dnnl.hpp"
24+
#include "dnnl_sycl.hpp"
25+
2526
class DnnlGemmWrapper {
2627
public:
2728
using dt = dnnl::memory::data_type;

0 commit comments

Comments
 (0)