Skip to content

Commit fadd59f

Browse files
Aviv HellerSaeed Mahameed
Aviv Heller
authored and
Saeed Mahameed
committed
net/mlx5: Introduce inter-device communication mechanism
This introduces devcom, a generic mechanism for performing operations on both physical functions of the same Connect-X card. The first user of this API is merged eswitch, which will be introduced in subsequent patches. Signed-off-by: Aviv Heller <[email protected]> Signed-off-by: Saeed Mahameed <[email protected]>
1 parent 64e4cf0 commit fadd59f

File tree

5 files changed

+313
-4
lines changed

5 files changed

+313
-4
lines changed

‎drivers/net/ethernet/mellanox/mlx5/core/Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ mlx5_core-y := main.o cmd.o debugfs.o fw.o eq.o uar.o pagealloc.o \
1515
health.o mcg.o cq.o alloc.o qp.o port.o mr.o pd.o \
1616
mad.o transobj.o vport.o sriov.o fs_cmd.o fs_core.o \
1717
fs_counters.o rl.o lag.o dev.o events.o wq.o lib/gid.o \
18-
diag/fs_tracepoint.o diag/fw_tracer.o
18+
lib/devcom.o diag/fs_tracepoint.o diag/fw_tracer.o
1919

2020
#
2121
# Netdev basic

‎drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c

Lines changed: 255 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,255 @@
1+
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2+
/* Copyright (c) 2018 Mellanox Technologies */
3+
4+
#include <linux/mlx5/vport.h>
5+
#include "lib/devcom.h"
6+
7+
static LIST_HEAD(devcom_list);
8+
9+
#define devcom_for_each_component(priv, comp, iter) \
10+
for (iter = 0; \
11+
comp = &(priv)->components[iter], iter < MLX5_DEVCOM_NUM_COMPONENTS; \
12+
iter++)
13+
14+
struct mlx5_devcom_component {
15+
struct {
16+
void *data;
17+
} device[MLX5_MAX_PORTS];
18+
19+
mlx5_devcom_event_handler_t handler;
20+
struct rw_semaphore sem;
21+
bool paired;
22+
};
23+
24+
struct mlx5_devcom_list {
25+
struct list_head list;
26+
27+
struct mlx5_devcom_component components[MLX5_DEVCOM_NUM_COMPONENTS];
28+
struct mlx5_core_dev *devs[MLX5_MAX_PORTS];
29+
};
30+
31+
struct mlx5_devcom {
32+
struct mlx5_devcom_list *priv;
33+
int idx;
34+
};
35+
36+
static struct mlx5_devcom_list *mlx5_devcom_list_alloc(void)
37+
{
38+
struct mlx5_devcom_component *comp;
39+
struct mlx5_devcom_list *priv;
40+
int i;
41+
42+
priv = kzalloc(sizeof(*priv), GFP_KERNEL);
43+
if (!priv)
44+
return NULL;
45+
46+
devcom_for_each_component(priv, comp, i)
47+
init_rwsem(&comp->sem);
48+
49+
return priv;
50+
}
51+
52+
static struct mlx5_devcom *mlx5_devcom_alloc(struct mlx5_devcom_list *priv,
53+
u8 idx)
54+
{
55+
struct mlx5_devcom *devcom;
56+
57+
devcom = kzalloc(sizeof(*devcom), GFP_KERNEL);
58+
if (!devcom)
59+
return NULL;
60+
61+
devcom->priv = priv;
62+
devcom->idx = idx;
63+
return devcom;
64+
}
65+
66+
/* Must be called with intf_mutex held */
67+
struct mlx5_devcom *mlx5_devcom_register_device(struct mlx5_core_dev *dev)
68+
{
69+
struct mlx5_devcom_list *priv = NULL, *iter;
70+
struct mlx5_devcom *devcom = NULL;
71+
bool new_priv = false;
72+
u64 sguid0, sguid1;
73+
int idx, i;
74+
75+
if (!mlx5_core_is_pf(dev))
76+
return NULL;
77+
78+
sguid0 = mlx5_query_nic_system_image_guid(dev);
79+
list_for_each_entry(iter, &devcom_list, list) {
80+
struct mlx5_core_dev *tmp_dev = NULL;
81+
82+
idx = -1;
83+
for (i = 0; i < MLX5_MAX_PORTS; i++) {
84+
if (iter->devs[i])
85+
tmp_dev = iter->devs[i];
86+
else
87+
idx = i;
88+
}
89+
90+
if (idx == -1)
91+
continue;
92+
93+
sguid1 = mlx5_query_nic_system_image_guid(tmp_dev);
94+
if (sguid0 != sguid1)
95+
continue;
96+
97+
priv = iter;
98+
break;
99+
}
100+
101+
if (!priv) {
102+
priv = mlx5_devcom_list_alloc();
103+
if (!priv)
104+
return ERR_PTR(-ENOMEM);
105+
106+
idx = 0;
107+
new_priv = true;
108+
}
109+
110+
priv->devs[idx] = dev;
111+
devcom = mlx5_devcom_alloc(priv, idx);
112+
if (!devcom) {
113+
kfree(priv);
114+
return ERR_PTR(-ENOMEM);
115+
}
116+
117+
if (new_priv)
118+
list_add(&priv->list, &devcom_list);
119+
120+
return devcom;
121+
}
122+
123+
/* Must be called with intf_mutex held */
124+
void mlx5_devcom_unregister_device(struct mlx5_devcom *devcom)
125+
{
126+
struct mlx5_devcom_list *priv;
127+
int i;
128+
129+
if (IS_ERR_OR_NULL(devcom))
130+
return;
131+
132+
priv = devcom->priv;
133+
priv->devs[devcom->idx] = NULL;
134+
135+
kfree(devcom);
136+
137+
for (i = 0; i < MLX5_MAX_PORTS; i++)
138+
if (priv->devs[i])
139+
break;
140+
141+
if (i != MLX5_MAX_PORTS)
142+
return;
143+
144+
list_del(&priv->list);
145+
kfree(priv);
146+
}
147+
148+
void mlx5_devcom_register_component(struct mlx5_devcom *devcom,
149+
enum mlx5_devcom_components id,
150+
mlx5_devcom_event_handler_t handler,
151+
void *data)
152+
{
153+
struct mlx5_devcom_component *comp;
154+
155+
if (IS_ERR_OR_NULL(devcom))
156+
return;
157+
158+
WARN_ON(!data);
159+
160+
comp = &devcom->priv->components[id];
161+
down_write(&comp->sem);
162+
comp->handler = handler;
163+
comp->device[devcom->idx].data = data;
164+
up_write(&comp->sem);
165+
}
166+
167+
void mlx5_devcom_unregister_component(struct mlx5_devcom *devcom,
168+
enum mlx5_devcom_components id)
169+
{
170+
struct mlx5_devcom_component *comp;
171+
172+
if (IS_ERR_OR_NULL(devcom))
173+
return;
174+
175+
comp = &devcom->priv->components[id];
176+
down_write(&comp->sem);
177+
comp->device[devcom->idx].data = NULL;
178+
up_write(&comp->sem);
179+
}
180+
181+
int mlx5_devcom_send_event(struct mlx5_devcom *devcom,
182+
enum mlx5_devcom_components id,
183+
int event,
184+
void *event_data)
185+
{
186+
struct mlx5_devcom_component *comp;
187+
int err = -ENODEV, i;
188+
189+
if (IS_ERR_OR_NULL(devcom))
190+
return err;
191+
192+
comp = &devcom->priv->components[id];
193+
down_write(&comp->sem);
194+
for (i = 0; i < MLX5_MAX_PORTS; i++)
195+
if (i != devcom->idx && comp->device[i].data) {
196+
err = comp->handler(event, comp->device[i].data,
197+
event_data);
198+
break;
199+
}
200+
201+
up_write(&comp->sem);
202+
return err;
203+
}
204+
205+
void mlx5_devcom_set_paired(struct mlx5_devcom *devcom,
206+
enum mlx5_devcom_components id,
207+
bool paired)
208+
{
209+
struct mlx5_devcom_component *comp;
210+
211+
comp = &devcom->priv->components[id];
212+
WARN_ON(!rwsem_is_locked(&comp->sem));
213+
214+
comp->paired = paired;
215+
}
216+
217+
bool mlx5_devcom_is_paired(struct mlx5_devcom *devcom,
218+
enum mlx5_devcom_components id)
219+
{
220+
if (IS_ERR_OR_NULL(devcom))
221+
return false;
222+
223+
return devcom->priv->components[id].paired;
224+
}
225+
226+
void *mlx5_devcom_get_peer_data(struct mlx5_devcom *devcom,
227+
enum mlx5_devcom_components id)
228+
{
229+
struct mlx5_devcom_component *comp;
230+
int i;
231+
232+
if (IS_ERR_OR_NULL(devcom))
233+
return NULL;
234+
235+
comp = &devcom->priv->components[id];
236+
down_read(&comp->sem);
237+
if (!comp->paired) {
238+
up_read(&comp->sem);
239+
return NULL;
240+
}
241+
242+
for (i = 0; i < MLX5_MAX_PORTS; i++)
243+
if (i != devcom->idx)
244+
break;
245+
246+
return comp->device[i].data;
247+
}
248+
249+
void mlx5_devcom_release_peer_data(struct mlx5_devcom *devcom,
250+
enum mlx5_devcom_components id)
251+
{
252+
struct mlx5_devcom_component *comp = &devcom->priv->components[id];
253+
254+
up_read(&comp->sem);
255+
}

‎drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
2+
/* Copyright (c) 2018 Mellanox Technologies */
3+
4+
#ifndef __LIB_MLX5_DEVCOM_H__
5+
#define __LIB_MLX5_DEVCOM_H__
6+
7+
#include <linux/mlx5/driver.h>
8+
9+
enum mlx5_devcom_components {
10+
MLX5_DEVCOM_NUM_COMPONENTS,
11+
};
12+
13+
typedef int (*mlx5_devcom_event_handler_t)(int event,
14+
void *my_data,
15+
void *event_data);
16+
17+
struct mlx5_devcom *mlx5_devcom_register_device(struct mlx5_core_dev *dev);
18+
void mlx5_devcom_unregister_device(struct mlx5_devcom *devcom);
19+
20+
void mlx5_devcom_register_component(struct mlx5_devcom *devcom,
21+
enum mlx5_devcom_components id,
22+
mlx5_devcom_event_handler_t handler,
23+
void *data);
24+
void mlx5_devcom_unregister_component(struct mlx5_devcom *devcom,
25+
enum mlx5_devcom_components id);
26+
27+
int mlx5_devcom_send_event(struct mlx5_devcom *devcom,
28+
enum mlx5_devcom_components id,
29+
int event,
30+
void *event_data);
31+
32+
void mlx5_devcom_set_paired(struct mlx5_devcom *devcom,
33+
enum mlx5_devcom_components id,
34+
bool paired);
35+
bool mlx5_devcom_is_paired(struct mlx5_devcom *devcom,
36+
enum mlx5_devcom_components id);
37+
38+
void *mlx5_devcom_get_peer_data(struct mlx5_devcom *devcom,
39+
enum mlx5_devcom_components id);
40+
void mlx5_devcom_release_peer_data(struct mlx5_devcom *devcom,
41+
enum mlx5_devcom_components id);
42+
43+
#endif
44+

‎drivers/net/ethernet/mellanox/mlx5/core/main.c

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@
6363
#include "accel/tls.h"
6464
#include "lib/clock.h"
6565
#include "lib/vxlan.h"
66+
#include "lib/devcom.h"
6667
#include "diag/fw_tracer.h"
6768

6869
MODULE_AUTHOR("Eli Cohen <[email protected]>");
@@ -722,16 +723,21 @@ static int mlx5_init_once(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
722723
struct pci_dev *pdev = dev->pdev;
723724
int err;
724725

726+
priv->devcom = mlx5_devcom_register_device(dev);
727+
if (IS_ERR(priv->devcom))
728+
dev_err(&pdev->dev, "failed to register with devcom (0x%p)\n",
729+
priv->devcom);
730+
725731
err = mlx5_query_board_id(dev);
726732
if (err) {
727733
dev_err(&pdev->dev, "query board id failed\n");
728-
goto out;
734+
goto err_devcom;
729735
}
730736

731737
err = mlx5_eq_table_init(dev);
732738
if (err) {
733739
dev_err(&pdev->dev, "failed to initialize eq\n");
734-
goto out;
740+
goto err_devcom;
735741
}
736742

737743
err = mlx5_events_init(dev);
@@ -807,8 +813,9 @@ static int mlx5_init_once(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
807813
mlx5_events_cleanup(dev);
808814
err_eq_cleanup:
809815
mlx5_eq_table_cleanup(dev);
816+
err_devcom:
817+
mlx5_devcom_unregister_device(dev->priv.devcom);
810818

811-
out:
812819
return err;
813820
}
814821

@@ -828,6 +835,7 @@ static void mlx5_cleanup_once(struct mlx5_core_dev *dev)
828835
mlx5_cq_debugfs_cleanup(dev);
829836
mlx5_events_cleanup(dev);
830837
mlx5_eq_table_cleanup(dev);
838+
mlx5_devcom_unregister_device(dev->priv.devcom);
831839
}
832840

833841
static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv,

‎include/linux/mlx5/driver.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -486,6 +486,7 @@ struct mlx5_events;
486486
struct mlx5_mpfs;
487487
struct mlx5_eswitch;
488488
struct mlx5_lag;
489+
struct mlx5_devcom;
489490
struct mlx5_eq_table;
490491

491492
struct mlx5_rate_limit {
@@ -560,6 +561,7 @@ struct mlx5_priv {
560561
struct mlx5_eswitch *eswitch;
561562
struct mlx5_core_sriov sriov;
562563
struct mlx5_lag *lag;
564+
struct mlx5_devcom *devcom;
563565
unsigned long pci_dev_data;
564566
struct mlx5_fc_stats fc_stats;
565567
struct mlx5_rl_table rl_table;

0 commit comments

Comments
 (0)