Skip to content

Commit 13a4f2d

Browse files
Dotan BarakSomasundaram Krishnasamy
authored andcommitted
mlx4_core: enable changing default max HCA resource limits.
Enable module-initialization time modification of default HCA maximum resource limits via module parameters, as is done in mthca. Specify the log of the parameter value, rather than the value itself to avoid the hidden side-effect of rounding up values to next power-of-2. For mtt's, there is a heuristic in place to configure twice the number of MTTs required to cover host memory. This heuristic is modified slightly to guarantee a minimum of 2^20 mtt's, and is used when the user does not set the log_num_mtts module parameter. If the log_num_mtt's module parameter is set by the user, the value set for log_num_mtts will be used as-is (instead of the heuristic). V2: Adapted for rebase to kernel 3.7-rc4 Signed-off-by: Jack Morgenstein <[email protected]> Signed-off-by: Dotan Barak <[email protected]> Reviewed-by: Jack Morgenstein <[email protected]> Signed-off-by: Vladimir Sokolovsky <[email protected]> Signed-off-by: Jack Morgenstein <[email protected]> (Ported from Mellanox OFED 2.4) Signed-off-by: Mukesh Kacker <[email protected]> Orabug: 27510510 (cherry picked from commit 3788851) cherry-pick-repo=linux-uek.git Conflicts: drivers/net/ethernet/mellanox/mlx4/main.c drivers/net/ethernet/mellanox/mlx4/profile.c Signed-off-by: Qing Huang <[email protected]> Signed-off-by: Aron Silverton <[email protected]> Signed-off-by: Somasundaram Krishnasamy <[email protected]>
1 parent 31f5e1f commit 13a4f2d

File tree

2 files changed

+74
-30
lines changed

2 files changed

+74
-30
lines changed

drivers/net/ethernet/mellanox/mlx4/main.c

Lines changed: 74 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -123,16 +123,6 @@ static char mlx4_version[] =
123123
DRV_NAME ": Mellanox ConnectX core driver v"
124124
DRV_VERSION "\n";
125125

126-
static const struct mlx4_profile default_profile = {
127-
.num_qp = 1 << 18,
128-
.num_srq = 1 << 16,
129-
.rdmarc_per_qp = 1 << 4,
130-
.num_cq = 1 << 16,
131-
.num_mcg = 1 << 13,
132-
.num_mpt = 1 << 19,
133-
.num_mtt = 1 << 20, /* It is really num mtt segements */
134-
};
135-
136126
static const struct mlx4_profile low_mem_profile = {
137127
.num_qp = 1 << 17,
138128
.num_srq = 1 << 6,
@@ -191,6 +181,79 @@ static inline void mlx4_set_num_reserved_uars(struct mlx4_dev *dev,
191181
(1 << (PAGE_SHIFT - dev->uar_page_shift)));
192182
}
193183

184+
#define MLX4_LOG_NUM_MTT 20
185+
static struct mlx4_profile mod_param_profile = {
186+
.num_qp = 18,
187+
.num_srq = 16,
188+
.rdmarc_per_qp = 4,
189+
.num_cq = 16,
190+
.num_mcg = 13,
191+
.num_mpt = 19,
192+
.num_mtt = 0, /* max(20, 2*MTTs for host memory)) */
193+
};
194+
195+
module_param_named(log_num_qp, mod_param_profile.num_qp, int, 0444);
196+
MODULE_PARM_DESC(log_num_qp, "log maximum number of QPs per HCA (default: 18)");
197+
198+
module_param_named(log_num_srq, mod_param_profile.num_srq, int, 0444);
199+
MODULE_PARM_DESC(log_num_srq, "log maximum number of SRQs per HCA "
200+
"(default: 16)");
201+
202+
module_param_named(log_rdmarc_per_qp, mod_param_profile.rdmarc_per_qp, int,
203+
0444);
204+
MODULE_PARM_DESC(log_rdmarc_per_qp, "log number of RDMARC buffers per QP "
205+
"(default: 4)");
206+
207+
module_param_named(log_num_cq, mod_param_profile.num_cq, int, 0444);
208+
MODULE_PARM_DESC(log_num_cq, "log maximum number of CQs per HCA (default: 16)");
209+
210+
module_param_named(log_num_mcg, mod_param_profile.num_mcg, int, 0444);
211+
MODULE_PARM_DESC(log_num_mcg, "log maximum number of multicast groups per HCA "
212+
"(default: 13)");
213+
214+
module_param_named(log_num_mpt, mod_param_profile.num_mpt, int, 0444);
215+
MODULE_PARM_DESC(log_num_mpt,
216+
"log maximum number of memory protection table entries per "
217+
"HCA (default: 19)");
218+
219+
module_param_named(log_num_mtt, mod_param_profile.num_mtt, int, 0444);
220+
MODULE_PARM_DESC(log_num_mtt,
221+
"log maximum number of memory translation table segments per "
222+
"HCA (default: max(20, 2*MTTs for register all of the host memory))");
223+
224+
static void process_mod_param_profile(struct mlx4_profile *profile)
225+
{
226+
struct sysinfo si;
227+
228+
profile->num_qp = 1 << mod_param_profile.num_qp;
229+
profile->num_srq = 1 << mod_param_profile.num_srq;
230+
profile->rdmarc_per_qp = 1 << mod_param_profile.rdmarc_per_qp;
231+
profile->num_cq = 1 << mod_param_profile.num_cq;
232+
profile->num_mcg = 1 << mod_param_profile.num_mcg;
233+
profile->num_mpt = 1 << mod_param_profile.num_mpt;
234+
/*
235+
* We want to scale the number of MTTs with the size of the
236+
* system memory, since it makes sense to register a lot of
237+
* memory on a system with a lot of memory. As a heuristic,
238+
* make sure we have enough MTTs to register twice the system
239+
* memory (with PAGE_SIZE entries).
240+
*
241+
* This number has to be a power of two and fit into 32 bits
242+
* due to device limitations, so cap this at 2^31 as well.
243+
* That limits us to 8TB of memory registration per HCA with
244+
* 4KB pages, which is probably OK for the next few months.
245+
*/
246+
if (mod_param_profile.num_mtt)
247+
profile->num_mtt = 1 << mod_param_profile.num_mtt;
248+
else {
249+
si_meminfo(&si);
250+
profile->num_mtt =
251+
roundup_pow_of_two(max_t(unsigned, 1 << MLX4_LOG_NUM_MTT,
252+
min(1UL << 31,
253+
si.totalram >> (log_mtts_per_seg - 1))));
254+
}
255+
}
256+
194257
int mlx4_check_port_params(struct mlx4_dev *dev,
195258
enum mlx4_port_type *port_type)
196259
{
@@ -2291,7 +2354,7 @@ static int mlx4_init_hca(struct mlx4_dev *dev)
22912354
mlx4_info(dev, "Running from within kdump kernel. Using low memory profile\n");
22922355
profile = low_mem_profile;
22932356
} else {
2294-
profile = default_profile;
2357+
process_mod_param_profile(&profile);
22952358
}
22962359
if (dev->caps.steering_mode ==
22972360
MLX4_STEERING_MODE_DEVICE_MANAGED)

drivers/net/ethernet/mellanox/mlx4/profile.c

Lines changed: 0 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -82,31 +82,12 @@ u64 mlx4_make_profile(struct mlx4_dev *dev,
8282

8383
u64 total_size = 0;
8484
struct mlx4_resource *profile;
85-
struct sysinfo si;
8685
int i, j;
8786

8887
profile = kcalloc(MLX4_RES_NUM, sizeof(*profile), GFP_KERNEL);
8988
if (!profile)
9089
return -ENOMEM;
9190

92-
/*
93-
* We want to scale the number of MTTs with the size of the
94-
* system memory, since it makes sense to register a lot of
95-
* memory on a system with a lot of memory. As a heuristic,
96-
* make sure we have enough MTTs to cover twice the system
97-
* memory (with PAGE_SIZE entries).
98-
*
99-
* This number has to be a power of two and fit into 32 bits
100-
* due to device limitations, so cap this at 2^31 as well.
101-
* That limits us to 8TB of memory registration per HCA with
102-
* 4KB pages, which is probably OK for the next few months.
103-
*/
104-
si_meminfo(&si);
105-
request->num_mtt =
106-
roundup_pow_of_two(max_t(unsigned, request->num_mtt,
107-
min(1UL << (31 - log_mtts_per_seg),
108-
si.totalram >> (log_mtts_per_seg - 1))));
109-
11091
profile[MLX4_RES_QP].size = dev_cap->qpc_entry_sz;
11192
profile[MLX4_RES_RDMARC].size = dev_cap->rdmarc_entry_sz;
11293
profile[MLX4_RES_ALTC].size = dev_cap->altc_entry_sz;

0 commit comments

Comments
 (0)