Skip to content

Commit 08a9ad8

Browse files
bvanasscheaxboe
bvanassche
authored and
axboe
committed
block/mq-deadline: Add cgroup support
Maintain statistics per cgroup and export these to user space. These statistics are essential for verifying whether the proper I/O priorities have been assigned to requests. An example of the statistics data with this patch applied: $ cat /sys/fs/cgroup/io.stat 11:2 rbytes=0 wbytes=0 rios=3 wios=0 dbytes=0 dios=0 [NONE] dispatched=0 inserted=0 merged=171 [RT] dispatched=0 inserted=0 merged=0 [BE] dispatched=0 inserted=0 merged=0 [IDLE] dispatched=0 inserted=0 merged=0 8:32 rbytes=2142720 wbytes=0 rios=105 wios=0 dbytes=0 dios=0 [NONE] dispatched=0 inserted=0 merged=171 [RT] dispatched=0 inserted=0 merged=0 [BE] dispatched=0 inserted=0 merged=0 [IDLE] dispatched=0 inserted=0 merged=0 Cc: Damien Le Moal <[email protected]> Cc: Hannes Reinecke <[email protected]> Cc: Christoph Hellwig <[email protected]> Cc: Ming Lei <[email protected]> Cc: Johannes Thumshirn <[email protected]> Cc: Himanshu Madhani <[email protected]> Signed-off-by: Bart Van Assche <[email protected]> Link: https://lore.kernel.org/r/[email protected] Signed-off-by: Jens Axboe <[email protected]>
1 parent 38ba64d commit 08a9ad8

File tree

5 files changed

+308
-14
lines changed

5 files changed

+308
-14
lines changed

‎block/Kconfig.iosched

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,12 @@ config MQ_IOSCHED_DEADLINE
99
help
1010
MQ version of the deadline IO scheduler.
1111

12+
config MQ_IOSCHED_DEADLINE_CGROUP
13+
tristate
14+
default y
15+
depends on MQ_IOSCHED_DEADLINE
16+
depends on BLK_CGROUP
17+
1218
config MQ_IOSCHED_KYBER
1319
tristate "Kyber I/O scheduler"
1420
default y

‎block/Makefile

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ obj-$(CONFIG_BLK_CGROUP_IOPRIO) += blk-ioprio.o
2121
obj-$(CONFIG_BLK_CGROUP_IOLATENCY) += blk-iolatency.o
2222
obj-$(CONFIG_BLK_CGROUP_IOCOST) += blk-iocost.o
2323
obj-$(CONFIG_MQ_IOSCHED_DEADLINE) += mq-deadline.o
24+
mq-deadline-y += mq-deadline-main.o
25+
mq-deadline-$(CONFIG_MQ_IOSCHED_DEADLINE_CGROUP)+= mq-deadline-cgroup.o
2426
obj-$(CONFIG_MQ_IOSCHED_KYBER) += kyber-iosched.o
2527
bfq-y := bfq-iosched.o bfq-wf2q.o bfq-cgroup.o
2628
obj-$(CONFIG_IOSCHED_BFQ) += bfq.o

‎block/mq-deadline-cgroup.c

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
// SPDX-License-Identifier: GPL-2.0
2+
3+
#include <linux/blk-cgroup.h>
4+
#include <linux/ioprio.h>
5+
6+
#include "mq-deadline-cgroup.h"
7+
8+
static struct blkcg_policy dd_blkcg_policy;
9+
10+
static struct blkcg_policy_data *dd_cpd_alloc(gfp_t gfp)
11+
{
12+
struct dd_blkcg *pd;
13+
14+
pd = kzalloc(sizeof(*pd), gfp);
15+
if (!pd)
16+
return NULL;
17+
pd->stats = alloc_percpu_gfp(typeof(*pd->stats),
18+
GFP_KERNEL | __GFP_ZERO);
19+
if (!pd->stats) {
20+
kfree(pd);
21+
return NULL;
22+
}
23+
return &pd->cpd;
24+
}
25+
26+
static void dd_cpd_free(struct blkcg_policy_data *cpd)
27+
{
28+
struct dd_blkcg *dd_blkcg = container_of(cpd, typeof(*dd_blkcg), cpd);
29+
30+
free_percpu(dd_blkcg->stats);
31+
kfree(dd_blkcg);
32+
}
33+
34+
static struct dd_blkcg *dd_blkcg_from_pd(struct blkg_policy_data *pd)
35+
{
36+
return container_of(blkcg_to_cpd(pd->blkg->blkcg, &dd_blkcg_policy),
37+
struct dd_blkcg, cpd);
38+
}
39+
40+
/*
41+
* Convert an association between a block cgroup and a request queue into a
42+
* pointer to the mq-deadline information associated with a (blkcg, queue) pair.
43+
*/
44+
struct dd_blkcg *dd_blkcg_from_bio(struct bio *bio)
45+
{
46+
struct blkg_policy_data *pd;
47+
48+
pd = blkg_to_pd(bio->bi_blkg, &dd_blkcg_policy);
49+
if (!pd)
50+
return NULL;
51+
52+
return dd_blkcg_from_pd(pd);
53+
}
54+
55+
static size_t dd_pd_stat(struct blkg_policy_data *pd, char *buf, size_t size)
56+
{
57+
static const char *const prio_class_name[] = {
58+
[IOPRIO_CLASS_NONE] = "NONE",
59+
[IOPRIO_CLASS_RT] = "RT",
60+
[IOPRIO_CLASS_BE] = "BE",
61+
[IOPRIO_CLASS_IDLE] = "IDLE",
62+
};
63+
struct dd_blkcg *blkcg = dd_blkcg_from_pd(pd);
64+
int res = 0;
65+
u8 prio;
66+
67+
for (prio = 0; prio < ARRAY_SIZE(blkcg->stats->stats); prio++)
68+
res += scnprintf(buf + res, size - res,
69+
" [%s] dispatched=%u inserted=%u merged=%u",
70+
prio_class_name[prio],
71+
ddcg_sum(blkcg, dispatched, prio) +
72+
ddcg_sum(blkcg, merged, prio) -
73+
ddcg_sum(blkcg, completed, prio),
74+
ddcg_sum(blkcg, inserted, prio) -
75+
ddcg_sum(blkcg, completed, prio),
76+
ddcg_sum(blkcg, merged, prio));
77+
78+
return res;
79+
}
80+
81+
static struct blkg_policy_data *dd_pd_alloc(gfp_t gfp, struct request_queue *q,
82+
struct blkcg *blkcg)
83+
{
84+
struct dd_blkg *pd;
85+
86+
pd = kzalloc(sizeof(*pd), gfp);
87+
if (!pd)
88+
return NULL;
89+
return &pd->pd;
90+
}
91+
92+
static void dd_pd_free(struct blkg_policy_data *pd)
93+
{
94+
struct dd_blkg *dd_blkg = container_of(pd, typeof(*dd_blkg), pd);
95+
96+
kfree(dd_blkg);
97+
}
98+
99+
static struct blkcg_policy dd_blkcg_policy = {
100+
.cpd_alloc_fn = dd_cpd_alloc,
101+
.cpd_free_fn = dd_cpd_free,
102+
103+
.pd_alloc_fn = dd_pd_alloc,
104+
.pd_free_fn = dd_pd_free,
105+
.pd_stat_fn = dd_pd_stat,
106+
};
107+
108+
int dd_activate_policy(struct request_queue *q)
109+
{
110+
return blkcg_activate_policy(q, &dd_blkcg_policy);
111+
}
112+
113+
void dd_deactivate_policy(struct request_queue *q)
114+
{
115+
blkcg_deactivate_policy(q, &dd_blkcg_policy);
116+
}
117+
118+
int __init dd_blkcg_init(void)
119+
{
120+
return blkcg_policy_register(&dd_blkcg_policy);
121+
}
122+
123+
void __exit dd_blkcg_exit(void)
124+
{
125+
blkcg_policy_unregister(&dd_blkcg_policy);
126+
}

‎block/mq-deadline-cgroup.h

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
/* SPDX-License-Identifier: GPL-2.0 */
2+
3+
#if !defined(_MQ_DEADLINE_CGROUP_H_)
4+
#define _MQ_DEADLINE_CGROUP_H_
5+
6+
#include <linux/blk-cgroup.h>
7+
8+
struct request_queue;
9+
10+
/**
11+
* struct io_stats_per_prio - I/O statistics per I/O priority class.
12+
* @inserted: Number of inserted requests.
13+
* @merged: Number of merged requests.
14+
* @dispatched: Number of dispatched requests.
15+
* @completed: Number of I/O completions.
16+
*/
17+
struct io_stats_per_prio {
18+
local_t inserted;
19+
local_t merged;
20+
local_t dispatched;
21+
local_t completed;
22+
};
23+
24+
/* I/O statistics per I/O cgroup per I/O priority class (IOPRIO_CLASS_*). */
25+
struct blkcg_io_stats {
26+
struct io_stats_per_prio stats[4];
27+
};
28+
29+
/**
30+
* struct dd_blkcg - Per cgroup data.
31+
* @cpd: blkcg_policy_data structure.
32+
* @stats: I/O statistics.
33+
*/
34+
struct dd_blkcg {
35+
struct blkcg_policy_data cpd; /* must be the first member */
36+
struct blkcg_io_stats __percpu *stats;
37+
};
38+
39+
/*
40+
* Count one event of type 'event_type' and with I/O priority class
41+
* 'prio_class'.
42+
*/
43+
#define ddcg_count(ddcg, event_type, prio_class) do { \
44+
if (ddcg) { \
45+
struct blkcg_io_stats *io_stats = get_cpu_ptr((ddcg)->stats); \
46+
\
47+
BUILD_BUG_ON(!__same_type((ddcg), struct dd_blkcg *)); \
48+
BUILD_BUG_ON(!__same_type((prio_class), u8)); \
49+
local_inc(&io_stats->stats[(prio_class)].event_type); \
50+
put_cpu_ptr(io_stats); \
51+
} \
52+
} while (0)
53+
54+
/*
55+
* Returns the total number of ddcg_count(ddcg, event_type, prio_class) calls
56+
* across all CPUs. No locking or barriers since it is fine if the returned
57+
* sum is slightly outdated.
58+
*/
59+
#define ddcg_sum(ddcg, event_type, prio) ({ \
60+
unsigned int cpu; \
61+
u32 sum = 0; \
62+
\
63+
BUILD_BUG_ON(!__same_type((ddcg), struct dd_blkcg *)); \
64+
BUILD_BUG_ON(!__same_type((prio), u8)); \
65+
for_each_present_cpu(cpu) \
66+
sum += local_read(&per_cpu_ptr((ddcg)->stats, cpu)-> \
67+
stats[(prio)].event_type); \
68+
sum; \
69+
})
70+
71+
#ifdef CONFIG_BLK_CGROUP
72+
73+
/**
74+
* struct dd_blkg - Per (cgroup, request queue) data.
75+
* @pd: blkg_policy_data structure.
76+
*/
77+
struct dd_blkg {
78+
struct blkg_policy_data pd; /* must be the first member */
79+
};
80+
81+
struct dd_blkcg *dd_blkcg_from_bio(struct bio *bio);
82+
int dd_activate_policy(struct request_queue *q);
83+
void dd_deactivate_policy(struct request_queue *q);
84+
int __init dd_blkcg_init(void);
85+
void __exit dd_blkcg_exit(void);
86+
87+
#else /* CONFIG_BLK_CGROUP */
88+
89+
static inline struct dd_blkcg *dd_blkcg_from_bio(struct bio *bio)
90+
{
91+
return NULL;
92+
}
93+
94+
static inline int dd_activate_policy(struct request_queue *q)
95+
{
96+
return 0;
97+
}
98+
99+
static inline void dd_deactivate_policy(struct request_queue *q)
100+
{
101+
}
102+
103+
static inline int dd_blkcg_init(void)
104+
{
105+
return 0;
106+
}
107+
108+
static inline void dd_blkcg_exit(void)
109+
{
110+
}
111+
112+
#endif /* CONFIG_BLK_CGROUP */
113+
114+
#endif /* _MQ_DEADLINE_CGROUP_H_ */

0 commit comments

Comments
 (0)