Skip to content

Commit 51beb6d

Browse files
mqymqy
committed
remove struct ggml_task_stage.valid, add GGML_BACKEND_UNKNOWN; mulmat-tune: add --backend option
1 parent d23a4a6 commit 51beb6d

22 files changed

+392
-417
lines changed

‎examples/mulmat-tune/README.md

Lines changed: 18 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,6 @@ enum ggml_backend {
131131

132132
// new
133133
struct ggml_task_stage {
134-
bool valid;
135134
int backend; // enum ggml_backend
136135
bool parallel;
137136
bool wait;
@@ -149,44 +148,6 @@ struct ggml_tensor {
149148
}
150149
```
151150

152-
**Setup task profiles for mul_mat**
153-
154-
```c
155-
// examples/mulmat-tune/mulmat-tune.c
156-
157-
static struct ggml_task_profile
158-
ggml_mulmat_task_profiles_f32[GGML_MULMAT_MAX_PROFILES];
159-
static struct ggml_task_profile
160-
ggml_mulmat_task_profiles_f16[GGML_MULMAT_MAX_PROFILES];
161-
static struct ggml_task_profile
162-
ggml_mulmat_task_profiles_qxx[GGML_MULMAT_MAX_PROFILES];
163-
164-
static int ggml_mulmat_task_profiles_f32_n;
165-
static int ggml_mulmat_task_profiles_f16_n;
166-
static int ggml_mulmat_task_profiles_qxx_n;
167-
168-
void ggml_mulmat_init_task_profiles(void) {
169-
// ...
170-
}
171-
172-
int ggml_mulmat_get_task_profiles(struct ggml_task_profile **profiles,
173-
int src0_type, int src1_type) {
174-
GGML_ASSERT(src1_type == GGML_TYPE_F32);
175-
if (src0_type == GGML_TYPE_F32) {
176-
*profiles = ggml_mulmat_task_profiles_f32;
177-
return ggml_mulmat_task_profiles_f32_n;
178-
} else if (src0_type == GGML_TYPE_F16) {
179-
*profiles = ggml_mulmat_task_profiles_f16;
180-
return ggml_mulmat_task_profiles_f32_n;
181-
} else if (ggml_is_quantized(src0_type)) {
182-
*profiles = ggml_mulmat_task_profiles_qxx;
183-
return ggml_mulmat_task_profiles_qxx_n;
184-
} else {
185-
GGML_ASSERT(false);
186-
}
187-
}
188-
```
189-
190151
## Limitations and TODOs
191152

192153
- Only tested models 7B and 13B.
@@ -266,20 +227,25 @@ The program will print debug log when or when not found the file.
266227
**Example**
267228

268229
```
269-
1 7B 2 Q4_0 3 ACCELERATE 4 8 16 3
270-
1 0 0 0 1 0 1 0 0 0 0 0
271-
0 0 0 0 1 3 0 1 0 0 0 0
272-
1 0 1 0 1 3 0 1 0 0 0 0
230+
$ ./mulmat-tune bench --m_num 2
231+
[BENCH] model: 7B, type: Q4_0, backend: CBLAS, BLAS vendor: ACCELERATE.
232+
233+
1 7B 2 Q4_0 3 ACCELERATE 4 16 2 3
234+
0 0 0 0 1 0 -1 0 0
235+
-1 0 0 3 0 1 -1 0 0
236+
0 1 0 3 0 1 -1 0 0
273237
4096 4096
274-
8 11 8477 0 0 5673 0 13547 5765 0
275-
16 23 14049 0 0 6331 0 13420 6516 0
276-
...
277-
128 154 110852 0 0 11529 0 11216 13137 0
238+
16 26 18636 0 0 0 0 15280 6139 0
239+
32 52 38288 0 0 0 0 15038 7565 0
278240
4096 11008
279-
...
241+
16 73 45606 0 0 0 0 30241 9104 0
242+
32 140 84720 0 0 0 0 35621 13743 0
280243
11008 4096
281-
...
244+
16 29 52994 0 0 0 0 47186 20651 0
245+
32 61 72840 0 0 0 0 36833 25601 0
282246
32000 4096
247+
16 26 113438 0 0 0 0 100848 66797 0
248+
32 37 207217 0 0 0 0 87275 74103 0
283249
...
284250
```
285251

@@ -308,9 +274,8 @@ m_step: 8
308274
m_num: 16
309275
310276
task_conf_profile: stage_conf(init) stage_conf(compute) stage_conf(finalize)
311-
stage_conf: valid backend parallel wait
312-
valid: 0 | 1
313-
backend: 0 | 1 | 2 | 3 | 4
277+
stage_conf: backend parallel wait
278+
backend: -1 (UNKNOWN) | 0 (CPU) | 1 (CUDA) | 2 (CL) | 3 (CBLAS)
314279
parallel: 0 | 1
315280
wait: 0 | 1
316281
@@ -321,6 +286,7 @@ profile_time := stage_time[3]
321286
stage_time[3]: init_time, compute_time, finalize_time
322287
```
323288

289+
A task stage is invalid if it's backend equals to `GGML_BACKEND_UNKNOWN`.
324290
Time unit is `us`. A column is all zeros when that stage does not exist.
325291

326292
## Estimate Execution Time

‎examples/mulmat-tune/bench-out/13b.q4_0.accelerate.512.txt

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
1 13B 2 Q4_0 3 ACCELERATE 4 32 16 2
2-
1 0 0 0 1 0 1 0 0 0 0 0
3-
1 0 1 0 1 3 0 1 0 0 0 0
2+
0 0 0 0 1 0 -1 0 0
3+
0 1 0 3 0 1 -1 0 0
44
5120 5120
55
32 46 42635 0 17674 11020 0
66
64 93 85913 0 17427 12877 0

‎examples/mulmat-tune/bench-out/13b.q4_0.accelerate.txt

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
1 13B 2 Q4_0 3 ACCELERATE 4 16 8 3
2-
1 0 0 0 1 0 1 0 0 0 0 0
3-
0 0 0 0 1 3 0 1 0 0 0 0
4-
1 0 1 0 1 3 0 1 0 0 0 0
2+
0 0 0 0 1 0 -1 0 0
3+
-1 0 0 3 0 1 -1 0 0
4+
0 1 0 3 0 1 -1 0 0
55
5120 5120
66
16 27 22566 0 0 31200 0 17495 9932 0
77
32 45 41745 0 0 29082 0 21490 15809 0

‎examples/mulmat-tune/bench-out/13b.q4_0.blis.txt

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
1 13B 2 Q4_0 3 BLIS 4 16 8 3
2-
1 0 0 0 1 0 1 0 0 0 0 0
3-
0 0 0 0 1 3 0 1 0 0 0 0
4-
1 0 1 0 1 3 0 1 0 0 0 0
2+
0 0 0 0 1 0 -1 0 0
3+
-1 0 0 3 0 1 -1 0 0
4+
0 1 0 3 0 1 -1 0 0
55
5120 5120
66
16 29 22204 0 0 37448 0 17346 19476 0
77
32 68 41478 0 0 49361 0 17576 30757 0

‎examples/mulmat-tune/bench-out/13b.q4_0.cl.txt

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
1 13B 2 Q4_0 2 CLBLAST 4 16 8 3
2-
1 0 0 0 1 0 1 0 0 0 0 0
3-
0 0 0 0 1 2 0 0 0 0 0 0
4-
1 0 1 0 1 2 0 1 0 0 0 0
2+
0 0 0 0 1 0 -1 0 0
3+
-1 0 0 3 0 1 -1 0 0
4+
0 1 0 3 0 1 -1 0 0
55
5120 5120
66
8 11 10559 0 0 68720 0 17473 47809 0
77
16 25 19993 0 0 58817 0 18175 44292 0

‎examples/mulmat-tune/bench-out/13b.q4_0.openblas.txt

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
1 13B 2 Q4_0 3 OPENBLAS 4 16 8 3
2-
1 0 0 0 1 0 1 0 0 0 0 0
3-
0 0 0 0 1 3 0 1 0 0 0 0
4-
1 0 1 0 1 3 0 1 0 0 0 0
2+
0 0 0 0 1 0 -1 0 0
3+
-1 0 0 3 0 1 -1 0 0
4+
0 1 0 3 0 1 -1 0 0
55
5120 5120
66
16 22 20782 0 0 30245 0 17584 9364 0
77
32 44 41106 0 0 31228 0 17313 11498 0

‎examples/mulmat-tune/bench-out/7b.f16.accelerate.txt

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
1 7B 1 F16 3 ACCELERATE 4 16 8 2
2-
0 0 0 0 1 0 1 0 0 0 0 0
3-
0 0 0 0 1 3 0 1 0 0 0 0
2+
-1 0 0 0 1 0 -1 0 0
3+
-1 0 0 3 0 1 -1 0 0
44
4096 4096
55
16 0 19288 0 0 13438 0
66
32 0 44321 0 0 13322 0

‎examples/mulmat-tune/bench-out/7b.f32.accelerate.txt

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
1 7B 0 F32 3 ACCELERATE 4 16 8 2
2-
0 0 0 0 1 0 1 0 0 0 0 0
3-
0 0 0 0 1 3 0 1 0 0 0 0
2+
-1 0 0 0 1 0 -1 0 0
3+
-1 0 0 3 0 1 -1 0 0
44
4096 4096
55
16 0 24628 0 0 7399 0
66
32 0 45917 0 0 5791 0

‎examples/mulmat-tune/bench-out/7b.q4_0.accelerate.512.txt

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
1 7B 2 Q4_0 3 ACCELERATE 4 32 16 2
2-
1 0 0 0 1 0 1 0 0 0 0 0
3-
1 0 1 0 1 3 0 1 0 0 0 0
2+
0 0 0 0 1 0 -1 0 0
3+
0 1 0 3 0 1 -1 0 0
44
4096 4096
55
32 45 27671 0 11210 7859 0
66
64 100 54379 0 11051 8748 0

‎examples/mulmat-tune/bench-out/7b.q4_0.accelerate.txt

Lines changed: 35 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -1,40 +1,40 @@
11
1 7B 2 Q4_0 3 ACCELERATE 4 16 8 3
2-
1 0 0 0 1 0 1 0 0 0 0 0
3-
0 0 0 0 1 3 0 0 0 0 0 0
4-
1 0 1 0 1 3 0 1 0 0 0 0
2+
0 0 0 0 1 0 -1 0 0
3+
-1 0 0 3 0 1 -1 0 0
4+
0 1 0 3 0 1 -1 0 0
55
4096 4096
6-
16 24 16017 0 0 18330 0 12878 6298 0
7-
32 36 26501 0 0 17342 0 12718 6816 0
8-
48 54 40971 0 0 18284 0 12683 7328 0
9-
64 74 54085 0 0 19863 0 11763 7797 0
10-
80 149 68751 0 0 20134 0 11536 9224 0
11-
96 113 84297 0 0 21526 0 11452 10350 0
12-
112 133 97258 0 0 22500 0 12810 11698 0
13-
128 169 111401 0 0 22655 0 11386 12157 0
6+
16 23 14046 0 0 0 0 11366 6297 0
7+
32 36 26793 0 0 0 0 11244 6201 0
8+
48 55 40187 0 0 0 0 11316 7811 0
9+
64 78 54450 0 0 0 0 11149 7859 0
10+
80 96 68095 0 0 0 0 11258 8748 0
11+
96 114 81588 0 0 0 0 11017 10248 0
12+
112 134 96596 0 0 0 0 11186 10506 0
13+
128 157 112871 0 0 0 0 11179 11887 0
1414
4096 11008
15-
16 83 35596 0 0 39994 0 29830 9072 0
16-
32 99 72352 0 0 40683 0 29820 10591 0
17-
48 158 109005 0 0 44930 0 29893 13174 0
18-
64 209 145554 0 0 46241 0 29499 15271 0
19-
80 273 184377 0 0 49900 0 29971 17526 0
20-
96 435 218057 0 0 53918 0 30023 19602 0
21-
112 418 255924 0 0 53719 0 30935 24615 0
22-
128 514 296851 0 0 56404 0 30028 25871 0
15+
16 55 36520 0 0 0 0 29851 9467 0
16+
32 103 73460 0 0 0 0 29815 11175 0
17+
48 173 109619 0 0 0 0 29870 13368 0
18+
64 206 147174 0 0 0 0 29571 16828 0
19+
80 289 178721 0 0 0 0 29895 18013 0
20+
96 343 219130 0 0 0 0 29633 21457 0
21+
112 550 257754 0 0 0 0 30342 23557 0
22+
128 594 298395 0 0 0 0 29683 24796 0
2323
11008 4096
24-
16 18 35649 0 0 52313 0 30588 20513 0
25-
32 49 70479 0 0 57137 0 30575 22031 0
26-
48 55 107438 0 0 58322 0 30355 23811 0
27-
64 90 147252 0 0 58251 0 29869 26485 0
28-
80 137 184252 0 0 60463 0 29609 29338 0
29-
96 115 224771 0 0 60814 0 29993 30985 0
30-
112 193 266329 0 0 66013 0 30189 33458 0
31-
128 238 304333 0 0 68255 0 29952 36076 0
24+
16 19 35077 0 0 0 0 30130 21051 0
25+
32 43 71844 0 0 0 0 29937 21740 0
26+
48 56 108664 0 0 0 0 30534 23017 0
27+
64 94 148288 0 0 0 0 29848 26486 0
28+
80 108 187098 0 0 0 0 29896 29687 0
29+
96 116 224466 0 0 0 0 29931 31416 0
30+
112 137 264372 0 0 0 0 29797 34035 0
31+
128 178 300958 0 0 0 0 29713 37036 0
3232
32000 4096
33-
16 18 106173 0 0 149332 0 87685 65974 0
34-
32 35 208572 0 0 152785 0 91938 73506 0
35-
48 61 336616 0 0 161289 0 87703 78153 0
36-
64 80 437456 0 0 165418 0 87400 84302 0
37-
80 94 553428 0 0 174794 0 87344 90493 0
38-
96 115 662666 0 0 178926 0 87326 96845 0
39-
112 199 775879 0 0 185675 0 86680 103695 0
40-
128 155 888381 0 0 196860 0 87641 111094 0
33+
16 18 105077 0 0 0 0 87731 67479 0
34+
32 36 205088 0 0 0 0 86620 72865 0
35+
48 54 314438 0 0 0 0 87458 77700 0
36+
64 75 420397 0 0 0 0 86515 83575 0
37+
80 109 541305 0 0 0 0 86580 88873 0
38+
96 121 646842 0 0 0 0 86500 96982 0
39+
112 134 761083 0 0 0 0 87326 102948 0
40+
128 155 872466 0 0 0 0 87668 112924 0

‎examples/mulmat-tune/bench-out/7b.q4_0.blis.txt

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
1 7B 2 Q4_0 3 BLIS 4 16 8 3
2-
1 0 0 0 1 0 1 0 0 0 0 0
3-
0 0 0 0 1 3 0 1 0 0 0 0
4-
1 0 1 0 1 3 0 1 0 0 0 0
2+
0 0 0 0 1 0 -1 0 0
3+
-1 0 0 3 0 1 -1 0 0
4+
0 1 0 3 0 1 -1 0 0
55
4096 4096
66
16 28 15279 0 0 24020 0 11232 12638 0
77
32 41 26418 0 0 30927 0 11188 20646 0

‎examples/mulmat-tune/bench-out/7b.q4_0.cl.txt

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
1 7B 2 Q4_0 2 CLBLAST 4 8 16 3
2-
1 0 0 0 1 0 1 0 0 0 0 0
3-
0 0 0 0 1 2 0 0 0 0 0 0
4-
1 0 1 0 1 2 0 1 0 0 0 0
2+
0 0 0 0 1 0 -1 0 0
3+
-1 0 0 3 0 1 -1 0 0
4+
0 1 0 3 0 1 -1 0 0
55
4096 4096
66
8 11 7125 0 0 49551 0 12217 35606 0
77
16 22 14451 0 0 48952 0 13487 35829 0

‎examples/mulmat-tune/bench-out/7b.q4_0.openblas.txt

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
1 7B 2 Q4_0 3 OPENBLAS 4 16 8 3
2-
1 0 0 0 1 0 1 0 0 0 0 0
3-
0 0 0 0 1 3 0 1 0 0 0 0
4-
1 0 1 0 1 3 0 1 0 0 0 0
2+
0 0 0 0 1 0 -1 0 0
3+
-1 0 0 3 0 1 -1 0 0
4+
0 1 0 3 0 1 -1 0 0
55
4096 4096
66
16 19 13040 0 0 18600 0 11032 6275 0
77
32 36 25982 0 0 18876 0 11212 8181 0

‎examples/mulmat-tune/bench-out/7b.q4_1.accelerate.txt

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
1 7B 3 Q4_1 3 ACCELERATE 4 16 8 3
2-
1 0 0 0 1 0 1 0 0 0 0 0
3-
0 0 0 0 1 3 0 1 0 0 0 0
4-
1 0 1 0 1 3 0 1 0 0 0 0
2+
0 0 0 0 1 0 -1 0 0
3+
-1 0 0 3 0 1 -1 0 0
4+
0 1 0 3 0 1 -1 0 0
55
4096 4096
66
16 29 15224 0 0 11816 0 6483 5609 0
77
32 46 26021 0 0 11732 0 5716 5528 0

‎examples/mulmat-tune/bench-out/7b.q5_0.accelerate.txt

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
1 7B 6 Q5_0 3 ACCELERATE 4 16 8 3
2-
1 0 0 0 1 0 1 0 0 0 0 0
3-
0 0 0 0 1 3 0 1 0 0 0 0
4-
1 0 1 0 1 3 0 1 0 0 0 0
2+
0 0 0 0 1 0 -1 0 0
3+
-1 0 0 3 0 1 -1 0 0
4+
0 1 0 3 0 1 -1 0 0
55
4096 4096
66
16 22 18804 0 0 13080 0 5569 5354 0
77
32 37 37371 0 0 11743 0 6491 6797 0

‎examples/mulmat-tune/bench-out/7b.q5_1.accelerate.txt

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
1 7B 7 Q5_1 3 ACCELERATE 4 16 8 3
2-
1 0 0 0 1 0 1 0 0 0 0 0
3-
0 0 0 0 1 3 0 1 0 0 0 0
4-
1 0 1 0 1 3 0 1 0 0 0 0
2+
0 0 0 0 1 0 -1 0 0
3+
-1 0 0 3 0 1 -1 0 0
4+
0 1 0 3 0 1 -1 0 0
55
4096 4096
66
16 32 18186 0 0 12092 0 6336 5943 0
77
32 69 35223 0 0 12135 0 5580 7008 0

‎examples/mulmat-tune/bench-out/7b.q8_0.accelerate.txt

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
1 7B 8 Q8_0 3 ACCELERATE 4 16 8 3
2-
1 0 0 0 1 0 1 0 0 0 0 0
3-
0 0 0 0 1 3 0 1 0 0 0 0
4-
1 0 1 0 1 3 0 1 0 0 0 0
2+
0 0 0 0 1 0 -1 0 0
3+
-1 0 0 3 0 1 -1 0 0
4+
0 1 0 3 0 1 -1 0 0
55
4096 4096
66
16 23 11976 0 0 11405 0 6945 4822 0
77
32 41 21502 0 0 11565 0 5619 6661 0

0 commit comments

Comments
 (0)