Skip to content

Commit 2e0cbc4

Browse files
Ram Amranidledford
Ram Amrani
authored and
dledford
committed
qedr: Add RoCE driver framework
Adds a skeletal implementation of the qed* RoCE driver - basically the ability to communicate with the qede driver and receive notifications from it regarding various init/exit events. Signed-off-by: Rajesh Borundia <[email protected]> Signed-off-by: Ram Amrani <[email protected]> Signed-off-by: Doug Ledford <[email protected]>
1 parent e9f8f48 commit 2e0cbc4

File tree

8 files changed

+331
-11
lines changed

8 files changed

+331
-11
lines changed

‎drivers/infiniband/Kconfig

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,4 +89,6 @@ source "drivers/infiniband/sw/rxe/Kconfig"
8989

9090
source "drivers/infiniband/hw/hfi1/Kconfig"
9191

92+
source "drivers/infiniband/hw/qedr/Kconfig"
93+
9294
endif # INFINIBAND

‎drivers/infiniband/hw/Makefile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,3 +10,4 @@ obj-$(CONFIG_INFINIBAND_OCRDMA) += ocrdma/
1010
obj-$(CONFIG_INFINIBAND_USNIC) += usnic/
1111
obj-$(CONFIG_INFINIBAND_HFI1) += hfi1/
1212
obj-$(CONFIG_INFINIBAND_HNS) += hns/
13+
obj-$(CONFIG_INFINIBAND_QEDR) += qedr/

‎drivers/infiniband/hw/qedr/Kconfig

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
config INFINIBAND_QEDR
2+
tristate "QLogic RoCE driver"
3+
depends on 64BIT && QEDE
4+
select QED_LL2
5+
---help---
6+
This driver provides low-level InfiniBand over Ethernet
7+
support for QLogic QED host channel adapters (HCAs).

‎drivers/infiniband/hw/qedr/Makefile

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
obj-$(CONFIG_INFINIBAND_QEDR) := qedr.o
2+
3+
qedr-y := main.o

‎drivers/infiniband/hw/qedr/main.c

Lines changed: 254 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,254 @@
1+
/* QLogic qedr NIC Driver
2+
* Copyright (c) 2015-2016 QLogic Corporation
3+
*
4+
* This software is available to you under a choice of one of two
5+
* licenses. You may choose to be licensed under the terms of the GNU
6+
* General Public License (GPL) Version 2, available from the file
7+
* COPYING in the main directory of this source tree, or the
8+
* OpenIB.org BSD license below:
9+
*
10+
* Redistribution and use in source and binary forms, with or
11+
* without modification, are permitted provided that the following
12+
* conditions are met:
13+
*
14+
* - Redistributions of source code must retain the above
15+
* copyright notice, this list of conditions and the following
16+
* disclaimer.
17+
*
18+
* - Redistributions in binary form must reproduce the above
19+
* copyright notice, this list of conditions and the following
20+
* disclaimer in the documentation and /or other materials
21+
* provided with the distribution.
22+
*
23+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24+
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25+
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26+
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27+
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28+
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29+
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30+
* SOFTWARE.
31+
*/
32+
#include <linux/module.h>
33+
#include <rdma/ib_verbs.h>
34+
#include <rdma/ib_addr.h>
35+
#include <linux/netdevice.h>
36+
#include <linux/iommu.h>
37+
#include <net/addrconf.h>
38+
#include <linux/qed/qede_roce.h>
39+
#include "qedr.h"
40+
41+
MODULE_DESCRIPTION("QLogic 40G/100G ROCE Driver");
42+
MODULE_AUTHOR("QLogic Corporation");
43+
MODULE_LICENSE("Dual BSD/GPL");
44+
MODULE_VERSION(QEDR_MODULE_VERSION);
45+
46+
void qedr_ib_dispatch_event(struct qedr_dev *dev, u8 port_num,
47+
enum ib_event_type type)
48+
{
49+
struct ib_event ibev;
50+
51+
ibev.device = &dev->ibdev;
52+
ibev.element.port_num = port_num;
53+
ibev.event = type;
54+
55+
ib_dispatch_event(&ibev);
56+
}
57+
58+
static enum rdma_link_layer qedr_link_layer(struct ib_device *device,
59+
u8 port_num)
60+
{
61+
return IB_LINK_LAYER_ETHERNET;
62+
}
63+
64+
static int qedr_register_device(struct qedr_dev *dev)
65+
{
66+
strlcpy(dev->ibdev.name, "qedr%d", IB_DEVICE_NAME_MAX);
67+
68+
memcpy(dev->ibdev.node_desc, QEDR_NODE_DESC, sizeof(QEDR_NODE_DESC));
69+
dev->ibdev.owner = THIS_MODULE;
70+
71+
dev->ibdev.get_link_layer = qedr_link_layer;
72+
73+
return 0;
74+
}
75+
76+
/* QEDR sysfs interface */
77+
static ssize_t show_rev(struct device *device, struct device_attribute *attr,
78+
char *buf)
79+
{
80+
struct qedr_dev *dev = dev_get_drvdata(device);
81+
82+
return scnprintf(buf, PAGE_SIZE, "0x%x\n", dev->pdev->vendor);
83+
}
84+
85+
static ssize_t show_hca_type(struct device *device,
86+
struct device_attribute *attr, char *buf)
87+
{
88+
return scnprintf(buf, PAGE_SIZE, "%s\n", "HCA_TYPE_TO_SET");
89+
}
90+
91+
static DEVICE_ATTR(hw_rev, S_IRUGO, show_rev, NULL);
92+
static DEVICE_ATTR(hca_type, S_IRUGO, show_hca_type, NULL);
93+
94+
static struct device_attribute *qedr_attributes[] = {
95+
&dev_attr_hw_rev,
96+
&dev_attr_hca_type
97+
};
98+
99+
static void qedr_remove_sysfiles(struct qedr_dev *dev)
100+
{
101+
int i;
102+
103+
for (i = 0; i < ARRAY_SIZE(qedr_attributes); i++)
104+
device_remove_file(&dev->ibdev.dev, qedr_attributes[i]);
105+
}
106+
107+
static void qedr_pci_set_atomic(struct qedr_dev *dev, struct pci_dev *pdev)
108+
{
109+
struct pci_dev *bridge;
110+
u32 val;
111+
112+
dev->atomic_cap = IB_ATOMIC_NONE;
113+
114+
bridge = pdev->bus->self;
115+
if (!bridge)
116+
return;
117+
118+
/* Check whether we are connected directly or via a switch */
119+
while (bridge && bridge->bus->parent) {
120+
DP_DEBUG(dev, QEDR_MSG_INIT,
121+
"Device is not connected directly to root. bridge->bus->number=%d primary=%d\n",
122+
bridge->bus->number, bridge->bus->primary);
123+
/* Need to check Atomic Op Routing Supported all the way to
124+
* root complex.
125+
*/
126+
pcie_capability_read_dword(bridge, PCI_EXP_DEVCAP2, &val);
127+
if (!(val & PCI_EXP_DEVCAP2_ATOMIC_ROUTE)) {
128+
pcie_capability_clear_word(pdev,
129+
PCI_EXP_DEVCTL2,
130+
PCI_EXP_DEVCTL2_ATOMIC_REQ);
131+
return;
132+
}
133+
bridge = bridge->bus->parent->self;
134+
}
135+
bridge = pdev->bus->self;
136+
137+
/* according to bridge capability */
138+
pcie_capability_read_dword(bridge, PCI_EXP_DEVCAP2, &val);
139+
if (val & PCI_EXP_DEVCAP2_ATOMIC_COMP64) {
140+
pcie_capability_set_word(pdev, PCI_EXP_DEVCTL2,
141+
PCI_EXP_DEVCTL2_ATOMIC_REQ);
142+
dev->atomic_cap = IB_ATOMIC_GLOB;
143+
} else {
144+
pcie_capability_clear_word(pdev, PCI_EXP_DEVCTL2,
145+
PCI_EXP_DEVCTL2_ATOMIC_REQ);
146+
}
147+
}
148+
149+
static struct qedr_dev *qedr_add(struct qed_dev *cdev, struct pci_dev *pdev,
150+
struct net_device *ndev)
151+
{
152+
struct qedr_dev *dev;
153+
int rc = 0, i;
154+
155+
dev = (struct qedr_dev *)ib_alloc_device(sizeof(*dev));
156+
if (!dev) {
157+
pr_err("Unable to allocate ib device\n");
158+
return NULL;
159+
}
160+
161+
DP_DEBUG(dev, QEDR_MSG_INIT, "qedr add device called\n");
162+
163+
dev->pdev = pdev;
164+
dev->ndev = ndev;
165+
dev->cdev = cdev;
166+
167+
qedr_pci_set_atomic(dev, pdev);
168+
169+
rc = qedr_register_device(dev);
170+
if (rc) {
171+
DP_ERR(dev, "Unable to allocate register device\n");
172+
goto init_err;
173+
}
174+
175+
for (i = 0; i < ARRAY_SIZE(qedr_attributes); i++)
176+
if (device_create_file(&dev->ibdev.dev, qedr_attributes[i]))
177+
goto init_err;
178+
179+
DP_DEBUG(dev, QEDR_MSG_INIT, "qedr driver loaded successfully\n");
180+
return dev;
181+
182+
init_err:
183+
ib_dealloc_device(&dev->ibdev);
184+
DP_ERR(dev, "qedr driver load failed rc=%d\n", rc);
185+
186+
return NULL;
187+
}
188+
189+
static void qedr_remove(struct qedr_dev *dev)
190+
{
191+
/* First unregister with stack to stop all the active traffic
192+
* of the registered clients.
193+
*/
194+
qedr_remove_sysfiles(dev);
195+
196+
ib_dealloc_device(&dev->ibdev);
197+
}
198+
199+
static int qedr_close(struct qedr_dev *dev)
200+
{
201+
qedr_ib_dispatch_event(dev, 1, IB_EVENT_PORT_ERR);
202+
203+
return 0;
204+
}
205+
206+
static void qedr_shutdown(struct qedr_dev *dev)
207+
{
208+
qedr_close(dev);
209+
qedr_remove(dev);
210+
}
211+
212+
/* event handling via NIC driver ensures that all the NIC specific
213+
* initialization done before RoCE driver notifies
214+
* event to stack.
215+
*/
216+
static void qedr_notify(struct qedr_dev *dev, enum qede_roce_event event)
217+
{
218+
switch (event) {
219+
case QEDE_UP:
220+
qedr_ib_dispatch_event(dev, 1, IB_EVENT_PORT_ACTIVE);
221+
break;
222+
case QEDE_DOWN:
223+
qedr_close(dev);
224+
break;
225+
case QEDE_CLOSE:
226+
qedr_shutdown(dev);
227+
break;
228+
case QEDE_CHANGE_ADDR:
229+
qedr_ib_dispatch_event(dev, 1, IB_EVENT_GID_CHANGE);
230+
break;
231+
default:
232+
pr_err("Event not supported\n");
233+
}
234+
}
235+
236+
static struct qedr_driver qedr_drv = {
237+
.name = "qedr_driver",
238+
.add = qedr_add,
239+
.remove = qedr_remove,
240+
.notify = qedr_notify,
241+
};
242+
243+
static int __init qedr_init_module(void)
244+
{
245+
return qede_roce_register_driver(&qedr_drv);
246+
}
247+
248+
static void __exit qedr_exit_module(void)
249+
{
250+
qede_roce_unregister_driver(&qedr_drv);
251+
}
252+
253+
module_init(qedr_init_module);
254+
module_exit(qedr_exit_module);

‎drivers/infiniband/hw/qedr/qedr.h

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
/* QLogic qedr NIC Driver
2+
* Copyright (c) 2015-2016 QLogic Corporation
3+
*
4+
* This software is available to you under a choice of one of two
5+
* licenses. You may choose to be licensed under the terms of the GNU
6+
* General Public License (GPL) Version 2, available from the file
7+
* COPYING in the main directory of this source tree, or the
8+
* OpenIB.org BSD license below:
9+
*
10+
* Redistribution and use in source and binary forms, with or
11+
* without modification, are permitted provided that the following
12+
* conditions are met:
13+
*
14+
* - Redistributions of source code must retain the above
15+
* copyright notice, this list of conditions and the following
16+
* disclaimer.
17+
*
18+
* - Redistributions in binary form must reproduce the above
19+
* copyright notice, this list of conditions and the following
20+
* disclaimer in the documentation and /or other materials
21+
* provided with the distribution.
22+
*
23+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24+
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25+
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26+
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27+
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28+
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29+
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30+
* SOFTWARE.
31+
*/
32+
#ifndef __QEDR_H__
33+
#define __QEDR_H__
34+
35+
#include <linux/pci.h>
36+
#include <rdma/ib_addr.h>
37+
#include <linux/qed/qed_if.h>
38+
#include <linux/qed/qede_roce.h>
39+
40+
#define QEDR_MODULE_VERSION "8.10.10.0"
41+
#define QEDR_NODE_DESC "QLogic 579xx RoCE HCA"
42+
#define DP_NAME(dev) ((dev)->ibdev.name)
43+
44+
#define DP_DEBUG(dev, module, fmt, ...) \
45+
pr_debug("(%s) " module ": " fmt, \
46+
DP_NAME(dev) ? DP_NAME(dev) : "", ## __VA_ARGS__)
47+
48+
#define QEDR_MSG_INIT "INIT"
49+
50+
struct qedr_dev {
51+
struct ib_device ibdev;
52+
struct qed_dev *cdev;
53+
struct pci_dev *pdev;
54+
struct net_device *ndev;
55+
56+
enum ib_atomic_cap atomic_cap;
57+
58+
u32 dp_module;
59+
u8 dp_level;
60+
};
61+
#endif

‎drivers/net/ethernet/qlogic/Kconfig

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -107,15 +107,4 @@ config QEDE
107107
---help---
108108
This enables the support for ...
109109

110-
config INFINIBAND_QEDR
111-
tristate "QLogic qede RoCE sources [debug]"
112-
depends on QEDE && 64BIT
113-
select QED_LL2
114-
default n
115-
---help---
116-
This provides a temporary node that allows the compilation
117-
and logical testing of the InfiniBand over Ethernet support
118-
for QLogic QED. This would be replaced by the 'real' option
119-
once the QEDR driver is added [+relocated].
120-
121110
endif # NET_VENDOR_QLOGIC

‎include/uapi/linux/pci_regs.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -612,13 +612,16 @@
612612
*/
613613
#define PCI_EXP_DEVCAP2 36 /* Device Capabilities 2 */
614614
#define PCI_EXP_DEVCAP2_ARI 0x00000020 /* Alternative Routing-ID */
615+
#define PCI_EXP_DEVCAP2_ATOMIC_ROUTE 0x00000040 /* Atomic Op routing */
616+
#define PCI_EXP_DEVCAP2_ATOMIC_COMP64 0x00000100 /* Atomic 64-bit compare */
615617
#define PCI_EXP_DEVCAP2_LTR 0x00000800 /* Latency tolerance reporting */
616618
#define PCI_EXP_DEVCAP2_OBFF_MASK 0x000c0000 /* OBFF support mechanism */
617619
#define PCI_EXP_DEVCAP2_OBFF_MSG 0x00040000 /* New message signaling */
618620
#define PCI_EXP_DEVCAP2_OBFF_WAKE 0x00080000 /* Re-use WAKE# for OBFF */
619621
#define PCI_EXP_DEVCTL2 40 /* Device Control 2 */
620622
#define PCI_EXP_DEVCTL2_COMP_TIMEOUT 0x000f /* Completion Timeout Value */
621623
#define PCI_EXP_DEVCTL2_ARI 0x0020 /* Alternative Routing-ID */
624+
#define PCI_EXP_DEVCTL2_ATOMIC_REQ 0x0040 /* Set Atomic requests */
622625
#define PCI_EXP_DEVCTL2_IDO_REQ_EN 0x0100 /* Allow IDO for requests */
623626
#define PCI_EXP_DEVCTL2_IDO_CMP_EN 0x0200 /* Allow IDO for completions */
624627
#define PCI_EXP_DEVCTL2_LTR_EN 0x0400 /* Enable LTR mechanism */

0 commit comments

Comments
 (0)