Skip to content

Commit 27bf91d

Browse files
Yevgeny PetrilinRoland Dreier
Yevgeny Petrilin
authored and
Roland Dreier
committed
mlx4_core: Add link type autosensing
When a port's link is down (except to driver restart) and the port is configured for auto sensing, we try to sense port link type (Ethernet or InfiniBand) in order to determine how to initialize the port. If the port type needs to be changed, all mlx4 for the device interfaces are unregistered and then registered again with the new port types. Sensing is done with intervals of 3 seconds. Signed-off-by: Yevgeny Petrilin <[email protected]> Signed-off-by: Roland Dreier <[email protected]>
1 parent 793730b commit 27bf91d

File tree

8 files changed

+277
-51
lines changed

8 files changed

+277
-51
lines changed

‎drivers/net/mlx4/Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
obj-$(CONFIG_MLX4_CORE) += mlx4_core.o
22

33
mlx4_core-y := alloc.o catas.o cmd.o cq.o eq.o fw.o icm.o intf.o main.o mcg.o \
4-
mr.o pd.o port.o profile.o qp.o reset.o srq.o
4+
mr.o pd.o port.o profile.o qp.o reset.o sense.o srq.o
55

66
obj-$(CONFIG_MLX4_EN) += mlx4_en.o
77

‎drivers/net/mlx4/catas.c

Lines changed: 2 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,6 @@ enum {
4242
static DEFINE_SPINLOCK(catas_lock);
4343

4444
static LIST_HEAD(catas_list);
45-
static struct workqueue_struct *catas_wq;
4645
static struct work_struct catas_work;
4746

4847
static int internal_err_reset = 1;
@@ -77,7 +76,7 @@ static void poll_catas(unsigned long dev_ptr)
7776
list_add(&priv->catas_err.list, &catas_list);
7877
spin_unlock(&catas_lock);
7978

80-
queue_work(catas_wq, &catas_work);
79+
queue_work(mlx4_wq, &catas_work);
8180
}
8281
} else
8382
mod_timer(&priv->catas_err.timer,
@@ -146,18 +145,7 @@ void mlx4_stop_catas_poll(struct mlx4_dev *dev)
146145
spin_unlock_irq(&catas_lock);
147146
}
148147

149-
int __init mlx4_catas_init(void)
148+
void __init mlx4_catas_init(void)
150149
{
151150
INIT_WORK(&catas_work, catas_reset);
152-
153-
catas_wq = create_singlethread_workqueue("mlx4_err");
154-
if (!catas_wq)
155-
return -ENOMEM;
156-
157-
return 0;
158-
}
159-
160-
void mlx4_catas_cleanup(void)
161-
{
162-
destroy_workqueue(catas_wq);
163151
}

‎drivers/net/mlx4/eq.c

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -163,6 +163,7 @@ static int mlx4_eq_int(struct mlx4_dev *dev, struct mlx4_eq *eq)
163163
int cqn;
164164
int eqes_found = 0;
165165
int set_ci = 0;
166+
int port;
166167

167168
while ((eqe = next_eqe_sw(eq))) {
168169
/*
@@ -203,11 +204,16 @@ static int mlx4_eq_int(struct mlx4_dev *dev, struct mlx4_eq *eq)
203204
break;
204205

205206
case MLX4_EVENT_TYPE_PORT_CHANGE:
206-
mlx4_dispatch_event(dev,
207-
eqe->subtype == MLX4_PORT_CHANGE_SUBTYPE_ACTIVE ?
208-
MLX4_DEV_EVENT_PORT_UP :
209-
MLX4_DEV_EVENT_PORT_DOWN,
210-
be32_to_cpu(eqe->event.port_change.port) >> 28);
207+
port = be32_to_cpu(eqe->event.port_change.port) >> 28;
208+
if (eqe->subtype == MLX4_PORT_CHANGE_SUBTYPE_DOWN) {
209+
mlx4_dispatch_event(dev, MLX4_DEV_EVENT_PORT_DOWN,
210+
port);
211+
mlx4_priv(dev)->sense.do_sense_port[port] = 1;
212+
} else {
213+
mlx4_dispatch_event(dev, MLX4_DEV_EVENT_PORT_UP,
214+
port);
215+
mlx4_priv(dev)->sense.do_sense_port[port] = 0;
216+
}
211217
break;
212218

213219
case MLX4_EVENT_TYPE_CQ_ERROR:

‎drivers/net/mlx4/main.c

Lines changed: 77 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,8 @@ MODULE_DESCRIPTION("Mellanox ConnectX HCA low-level driver");
5151
MODULE_LICENSE("Dual BSD/GPL");
5252
MODULE_VERSION(DRV_VERSION);
5353

54+
struct workqueue_struct *mlx4_wq;
55+
5456
#ifdef CONFIG_MLX4_DEBUG
5557

5658
int mlx4_debug_level = 0;
@@ -98,24 +100,23 @@ module_param_named(use_prio, use_prio, bool, 0444);
98100
MODULE_PARM_DESC(use_prio, "Enable steering by VLAN priority on ETH ports "
99101
"(0/1, default 0)");
100102

101-
static int mlx4_check_port_params(struct mlx4_dev *dev,
102-
enum mlx4_port_type *port_type)
103+
int mlx4_check_port_params(struct mlx4_dev *dev,
104+
enum mlx4_port_type *port_type)
103105
{
104106
int i;
105107

106108
for (i = 0; i < dev->caps.num_ports - 1; i++) {
107-
if (port_type[i] != port_type[i+1] &&
108-
!(dev->caps.flags & MLX4_DEV_CAP_FLAG_DPDP)) {
109-
mlx4_err(dev, "Only same port types supported "
110-
"on this HCA, aborting.\n");
111-
return -EINVAL;
109+
if (port_type[i] != port_type[i + 1]) {
110+
if (!(dev->caps.flags & MLX4_DEV_CAP_FLAG_DPDP)) {
111+
mlx4_err(dev, "Only same port types supported "
112+
"on this HCA, aborting.\n");
113+
return -EINVAL;
114+
}
115+
if (port_type[i] == MLX4_PORT_TYPE_ETH &&
116+
port_type[i + 1] == MLX4_PORT_TYPE_IB)
117+
return -EINVAL;
112118
}
113119
}
114-
if ((port_type[0] == MLX4_PORT_TYPE_ETH) &&
115-
(port_type[1] == MLX4_PORT_TYPE_IB)) {
116-
mlx4_err(dev, "eth-ib configuration is not supported.\n");
117-
return -EINVAL;
118-
}
119120

120121
for (i = 0; i < dev->caps.num_ports; i++) {
121122
if (!(port_type[i] & dev->caps.supported_type[i+1])) {
@@ -225,6 +226,9 @@ static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
225226
dev->caps.port_type[i] = MLX4_PORT_TYPE_IB;
226227
else
227228
dev->caps.port_type[i] = MLX4_PORT_TYPE_ETH;
229+
dev->caps.possible_type[i] = dev->caps.port_type[i];
230+
mlx4_priv(dev)->sense.sense_allowed[i] =
231+
dev->caps.supported_type[i] == MLX4_PORT_TYPE_AUTO;
228232

229233
if (dev->caps.log_num_macs > dev_cap->log_max_macs[i]) {
230234
dev->caps.log_num_macs = dev_cap->log_max_macs[i];
@@ -263,14 +267,16 @@ static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
263267
* Change the port configuration of the device.
264268
* Every user of this function must hold the port mutex.
265269
*/
266-
static int mlx4_change_port_types(struct mlx4_dev *dev,
267-
enum mlx4_port_type *port_types)
270+
int mlx4_change_port_types(struct mlx4_dev *dev,
271+
enum mlx4_port_type *port_types)
268272
{
269273
int err = 0;
270274
int change = 0;
271275
int port;
272276

273277
for (port = 0; port < dev->caps.num_ports; port++) {
278+
/* Change the port type only if the new type is different
279+
* from the current, and not set to Auto */
274280
if (port_types[port] != dev->caps.port_type[port + 1]) {
275281
change = 1;
276282
dev->caps.port_type[port + 1] = port_types[port];
@@ -302,10 +308,17 @@ static ssize_t show_port_type(struct device *dev,
302308
struct mlx4_port_info *info = container_of(attr, struct mlx4_port_info,
303309
port_attr);
304310
struct mlx4_dev *mdev = info->dev;
311+
char type[8];
312+
313+
sprintf(type, "%s",
314+
(mdev->caps.port_type[info->port] == MLX4_PORT_TYPE_IB) ?
315+
"ib" : "eth");
316+
if (mdev->caps.possible_type[info->port] == MLX4_PORT_TYPE_AUTO)
317+
sprintf(buf, "auto (%s)\n", type);
318+
else
319+
sprintf(buf, "%s\n", type);
305320

306-
return sprintf(buf, "%s\n",
307-
mdev->caps.port_type[info->port] == MLX4_PORT_TYPE_IB ?
308-
"ib" : "eth");
321+
return strlen(buf);
309322
}
310323

311324
static ssize_t set_port_type(struct device *dev,
@@ -317,33 +330,64 @@ static ssize_t set_port_type(struct device *dev,
317330
struct mlx4_dev *mdev = info->dev;
318331
struct mlx4_priv *priv = mlx4_priv(mdev);
319332
enum mlx4_port_type types[MLX4_MAX_PORTS];
333+
enum mlx4_port_type new_types[MLX4_MAX_PORTS];
320334
int i;
321335
int err = 0;
322336

323337
if (!strcmp(buf, "ib\n"))
324338
info->tmp_type = MLX4_PORT_TYPE_IB;
325339
else if (!strcmp(buf, "eth\n"))
326340
info->tmp_type = MLX4_PORT_TYPE_ETH;
341+
else if (!strcmp(buf, "auto\n"))
342+
info->tmp_type = MLX4_PORT_TYPE_AUTO;
327343
else {
328344
mlx4_err(mdev, "%s is not supported port type\n", buf);
329345
return -EINVAL;
330346
}
331347

348+
mlx4_stop_sense(mdev);
332349
mutex_lock(&priv->port_mutex);
333-
for (i = 0; i < mdev->caps.num_ports; i++)
350+
/* Possible type is always the one that was delivered */
351+
mdev->caps.possible_type[info->port] = info->tmp_type;
352+
353+
for (i = 0; i < mdev->caps.num_ports; i++) {
334354
types[i] = priv->port[i+1].tmp_type ? priv->port[i+1].tmp_type :
335-
mdev->caps.port_type[i+1];
355+
mdev->caps.possible_type[i+1];
356+
if (types[i] == MLX4_PORT_TYPE_AUTO)
357+
types[i] = mdev->caps.port_type[i+1];
358+
}
336359

337-
err = mlx4_check_port_params(mdev, types);
360+
if (!(mdev->caps.flags & MLX4_DEV_CAP_FLAG_DPDP)) {
361+
for (i = 1; i <= mdev->caps.num_ports; i++) {
362+
if (mdev->caps.possible_type[i] == MLX4_PORT_TYPE_AUTO) {
363+
mdev->caps.possible_type[i] = mdev->caps.port_type[i];
364+
err = -EINVAL;
365+
}
366+
}
367+
}
368+
if (err) {
369+
mlx4_err(mdev, "Auto sensing is not supported on this HCA. "
370+
"Set only 'eth' or 'ib' for both ports "
371+
"(should be the same)\n");
372+
goto out;
373+
}
374+
375+
mlx4_do_sense_ports(mdev, new_types, types);
376+
377+
err = mlx4_check_port_params(mdev, new_types);
338378
if (err)
339379
goto out;
340380

341-
for (i = 1; i <= mdev->caps.num_ports; i++)
342-
priv->port[i].tmp_type = 0;
381+
/* We are about to apply the changes after the configuration
382+
* was verified, no need to remember the temporary types
383+
* any more */
384+
for (i = 0; i < mdev->caps.num_ports; i++)
385+
priv->port[i + 1].tmp_type = 0;
343386

344-
err = mlx4_change_port_types(mdev, types);
387+
err = mlx4_change_port_types(mdev, new_types);
345388

346389
out:
390+
mlx4_start_sense(mdev);
347391
mutex_unlock(&priv->port_mutex);
348392
return err ? err : count;
349393
}
@@ -1117,6 +1161,9 @@ static int __mlx4_init_one(struct pci_dev *pdev, const struct pci_device_id *id)
11171161
if (err)
11181162
goto err_port;
11191163

1164+
mlx4_sense_init(dev);
1165+
mlx4_start_sense(dev);
1166+
11201167
pci_set_drvdata(pdev, dev);
11211168

11221169
return 0;
@@ -1182,6 +1229,7 @@ static void mlx4_remove_one(struct pci_dev *pdev)
11821229
int p;
11831230

11841231
if (dev) {
1232+
mlx4_stop_sense(dev);
11851233
mlx4_unregister_device(dev);
11861234

11871235
for (p = 1; p <= dev->caps.num_ports; p++) {
@@ -1266,9 +1314,11 @@ static int __init mlx4_init(void)
12661314
if (mlx4_verify_params())
12671315
return -EINVAL;
12681316

1269-
ret = mlx4_catas_init();
1270-
if (ret)
1271-
return ret;
1317+
mlx4_catas_init();
1318+
1319+
mlx4_wq = create_singlethread_workqueue("mlx4");
1320+
if (!mlx4_wq)
1321+
return -ENOMEM;
12721322

12731323
ret = pci_register_driver(&mlx4_driver);
12741324
return ret < 0 ? ret : 0;
@@ -1277,7 +1327,7 @@ static int __init mlx4_init(void)
12771327
static void __exit mlx4_cleanup(void)
12781328
{
12791329
pci_unregister_driver(&mlx4_driver);
1280-
mlx4_catas_cleanup();
1330+
destroy_workqueue(mlx4_wq);
12811331
}
12821332

12831333
module_init(mlx4_init);

‎drivers/net/mlx4/mlx4.h

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
#include <linux/mutex.h>
4141
#include <linux/radix-tree.h>
4242
#include <linux/timer.h>
43+
#include <linux/workqueue.h>
4344

4445
#include <linux/mlx4/device.h>
4546
#include <linux/mlx4/driver.h>
@@ -276,6 +277,13 @@ struct mlx4_port_info {
276277
struct mlx4_vlan_table vlan_table;
277278
};
278279

280+
struct mlx4_sense {
281+
struct mlx4_dev *dev;
282+
u8 do_sense_port[MLX4_MAX_PORTS + 1];
283+
u8 sense_allowed[MLX4_MAX_PORTS + 1];
284+
struct delayed_work sense_poll;
285+
};
286+
279287
struct mlx4_priv {
280288
struct mlx4_dev dev;
281289

@@ -305,6 +313,7 @@ struct mlx4_priv {
305313
struct mlx4_uar driver_uar;
306314
void __iomem *kar;
307315
struct mlx4_port_info port[MLX4_MAX_PORTS + 1];
316+
struct mlx4_sense sense;
308317
struct mutex port_mutex;
309318
};
310319

@@ -313,6 +322,10 @@ static inline struct mlx4_priv *mlx4_priv(struct mlx4_dev *dev)
313322
return container_of(dev, struct mlx4_priv, dev);
314323
}
315324

325+
#define MLX4_SENSE_RANGE (HZ * 3)
326+
327+
extern struct workqueue_struct *mlx4_wq;
328+
316329
u32 mlx4_bitmap_alloc(struct mlx4_bitmap *bitmap);
317330
void mlx4_bitmap_free(struct mlx4_bitmap *bitmap, u32 obj);
318331
u32 mlx4_bitmap_alloc_range(struct mlx4_bitmap *bitmap, int cnt, int align);
@@ -346,8 +359,7 @@ void mlx4_cleanup_mcg_table(struct mlx4_dev *dev);
346359

347360
void mlx4_start_catas_poll(struct mlx4_dev *dev);
348361
void mlx4_stop_catas_poll(struct mlx4_dev *dev);
349-
int mlx4_catas_init(void);
350-
void mlx4_catas_cleanup(void);
362+
void mlx4_catas_init(void);
351363
int mlx4_restart_one(struct pci_dev *pdev);
352364
int mlx4_register_device(struct mlx4_dev *dev);
353365
void mlx4_unregister_device(struct mlx4_dev *dev);
@@ -379,6 +391,17 @@ void mlx4_srq_event(struct mlx4_dev *dev, u32 srqn, int event_type);
379391

380392
void mlx4_handle_catas_err(struct mlx4_dev *dev);
381393

394+
void mlx4_do_sense_ports(struct mlx4_dev *dev,
395+
enum mlx4_port_type *stype,
396+
enum mlx4_port_type *defaults);
397+
void mlx4_start_sense(struct mlx4_dev *dev);
398+
void mlx4_stop_sense(struct mlx4_dev *dev);
399+
void mlx4_sense_init(struct mlx4_dev *dev);
400+
int mlx4_check_port_params(struct mlx4_dev *dev,
401+
enum mlx4_port_type *port_type);
402+
int mlx4_change_port_types(struct mlx4_dev *dev,
403+
enum mlx4_port_type *port_types);
404+
382405
void mlx4_init_mac_table(struct mlx4_dev *dev, struct mlx4_mac_table *table);
383406
void mlx4_init_vlan_table(struct mlx4_dev *dev, struct mlx4_vlan_table *table);
384407

0 commit comments

Comments
 (0)