Skip to content

Commit b5ca15a

Browse files
mark-blochSaeed Mahameed
mark-bloch
authored and
Saeed Mahameed
committed
IB/mlx5: Add proper representors support
This commit adds full support for IB representor: 1) Representors profile, We add two new profiles: nic_rep_profile - This profile will be used to create an IB device that represents the PF/UPLINK. rep_profile - This profile will be used to create an IB device that represents VFs. Each VF will be its own representor. 2) Proper load/unload callbacks, Those are called by the E-Switch when moving to/from switchdev mode. 3) Different flow DB handling for when we in switchdev mode. Signed-off-by: Mark Bloch <[email protected]> Signed-off-by: Leon Romanovsky <[email protected]> Signed-off-by: Saeed Mahameed <[email protected]>
1 parent b96c9dd commit b5ca15a

File tree

4 files changed

+192
-30
lines changed

4 files changed

+192
-30
lines changed

‎drivers/infiniband/hw/mlx5/ib_rep.c

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,42 @@
55

66
#include "ib_rep.h"
77

8+
static const struct mlx5_ib_profile rep_profile = {
9+
STAGE_CREATE(MLX5_IB_STAGE_INIT,
10+
mlx5_ib_stage_init_init,
11+
mlx5_ib_stage_init_cleanup),
12+
STAGE_CREATE(MLX5_IB_STAGE_FLOW_DB,
13+
mlx5_ib_stage_rep_flow_db_init,
14+
NULL),
15+
STAGE_CREATE(MLX5_IB_STAGE_CAPS,
16+
mlx5_ib_stage_caps_init,
17+
NULL),
18+
STAGE_CREATE(MLX5_IB_STAGE_NON_DEFAULT_CB,
19+
mlx5_ib_stage_rep_non_default_cb,
20+
NULL),
21+
STAGE_CREATE(MLX5_IB_STAGE_ROCE,
22+
mlx5_ib_stage_rep_roce_init,
23+
mlx5_ib_stage_rep_roce_cleanup),
24+
STAGE_CREATE(MLX5_IB_STAGE_DEVICE_RESOURCES,
25+
mlx5_ib_stage_dev_res_init,
26+
mlx5_ib_stage_dev_res_cleanup),
27+
STAGE_CREATE(MLX5_IB_STAGE_COUNTERS,
28+
mlx5_ib_stage_counters_init,
29+
mlx5_ib_stage_counters_cleanup),
30+
STAGE_CREATE(MLX5_IB_STAGE_BFREG,
31+
mlx5_ib_stage_bfrag_init,
32+
mlx5_ib_stage_bfrag_cleanup),
33+
STAGE_CREATE(MLX5_IB_STAGE_IB_REG,
34+
mlx5_ib_stage_ib_reg_init,
35+
mlx5_ib_stage_ib_reg_cleanup),
36+
STAGE_CREATE(MLX5_IB_STAGE_UMR_RESOURCES,
37+
mlx5_ib_stage_umr_res_init,
38+
mlx5_ib_stage_umr_res_cleanup),
39+
STAGE_CREATE(MLX5_IB_STAGE_CLASS_ATTR,
40+
mlx5_ib_stage_class_attr_init,
41+
NULL),
42+
};
43+
844
static int
945
mlx5_ib_nic_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep)
1046
{
@@ -14,17 +50,41 @@ mlx5_ib_nic_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep)
1450
static void
1551
mlx5_ib_nic_rep_unload(struct mlx5_eswitch_rep *rep)
1652
{
53+
rep->rep_if[REP_IB].priv = NULL;
1754
}
1855

1956
static int
2057
mlx5_ib_vport_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep)
2158
{
59+
struct mlx5_ib_dev *ibdev;
60+
61+
ibdev = (struct mlx5_ib_dev *)ib_alloc_device(sizeof(*ibdev));
62+
if (!ibdev)
63+
return -ENOMEM;
64+
65+
ibdev->rep = rep;
66+
ibdev->mdev = dev;
67+
ibdev->num_ports = max(MLX5_CAP_GEN(dev, num_ports),
68+
MLX5_CAP_GEN(dev, num_vhca_ports));
69+
if (!__mlx5_ib_add(ibdev, &rep_profile))
70+
return -EINVAL;
71+
72+
rep->rep_if[REP_IB].priv = ibdev;
73+
2274
return 0;
2375
}
2476

2577
static void
2678
mlx5_ib_vport_rep_unload(struct mlx5_eswitch_rep *rep)
2779
{
80+
struct mlx5_ib_dev *dev;
81+
82+
if (!rep->rep_if[REP_IB].priv)
83+
return;
84+
85+
dev = mlx5_ib_rep_to_dev(rep);
86+
__mlx5_ib_remove(dev, dev->profile, MLX5_IB_STAGE_MAX);
87+
rep->rep_if[REP_IB].priv = NULL;
2888
}
2989

3090
static void *mlx5_ib_vport_get_proto_dev(struct mlx5_eswitch_rep *rep)
@@ -98,6 +158,11 @@ struct net_device *mlx5_ib_get_rep_netdev(struct mlx5_eswitch *esw,
98158
return mlx5_eswitch_get_proto_dev(esw, vport_index, REP_ETH);
99159
}
100160

161+
struct mlx5_ib_dev *mlx5_ib_get_uplink_ibdev(struct mlx5_eswitch *esw)
162+
{
163+
return mlx5_eswitch_uplink_get_proto_dev(esw, REP_IB);
164+
}
165+
101166
struct mlx5_eswitch_rep *mlx5_ib_vport_rep(struct mlx5_eswitch *esw, int vport)
102167
{
103168
return mlx5_eswitch_vport_rep(esw, vport);

‎drivers/infiniband/hw/mlx5/ib_rep.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
u8 mlx5_ib_eswitch_mode(struct mlx5_eswitch *esw);
1414
struct mlx5_ib_dev *mlx5_ib_get_rep_ibdev(struct mlx5_eswitch *esw,
1515
int vport_index);
16+
struct mlx5_ib_dev *mlx5_ib_get_uplink_ibdev(struct mlx5_eswitch *esw);
1617
struct mlx5_eswitch_rep *mlx5_ib_vport_rep(struct mlx5_eswitch *esw,
1718
int vport_index);
1819
void mlx5_ib_register_vport_reps(struct mlx5_ib_dev *dev);
@@ -34,6 +35,12 @@ struct mlx5_ib_dev *mlx5_ib_get_rep_ibdev(struct mlx5_eswitch *esw,
3435
return NULL;
3536
}
3637

38+
static inline
39+
struct mlx5_ib_dev *mlx5_ib_get_uplink_ibdev(struct mlx5_eswitch *esw)
40+
{
41+
return NULL;
42+
}
43+
3744
static inline
3845
struct mlx5_eswitch_rep *mlx5_ib_vport_rep(struct mlx5_eswitch *esw,
3946
int vport_index)

‎drivers/infiniband/hw/mlx5/main.c

Lines changed: 95 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -4554,7 +4554,7 @@ static void mlx5_ib_cleanup_multiport_master(struct mlx5_ib_dev *dev)
45544554
mlx5_nic_vport_disable_roce(dev->mdev);
45554555
}
45564556

4557-
static void mlx5_ib_stage_init_cleanup(struct mlx5_ib_dev *dev)
4557+
void mlx5_ib_stage_init_cleanup(struct mlx5_ib_dev *dev)
45584558
{
45594559
mlx5_ib_cleanup_multiport_master(dev);
45604560
#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
@@ -4563,7 +4563,7 @@ static void mlx5_ib_stage_init_cleanup(struct mlx5_ib_dev *dev)
45634563
kfree(dev->port);
45644564
}
45654565

4566-
static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev)
4566+
int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev)
45674567
{
45684568
struct mlx5_core_dev *mdev = dev->mdev;
45694569
const char *name;
@@ -4647,12 +4647,26 @@ static int mlx5_ib_stage_flow_db_init(struct mlx5_ib_dev *dev)
46474647
return 0;
46484648
}
46494649

4650+
int mlx5_ib_stage_rep_flow_db_init(struct mlx5_ib_dev *dev)
4651+
{
4652+
struct mlx5_ib_dev *nic_dev;
4653+
4654+
nic_dev = mlx5_ib_get_uplink_ibdev(dev->mdev->priv.eswitch);
4655+
4656+
if (!nic_dev)
4657+
return -EINVAL;
4658+
4659+
dev->flow_db = nic_dev->flow_db;
4660+
4661+
return 0;
4662+
}
4663+
46504664
static void mlx5_ib_stage_flow_db_cleanup(struct mlx5_ib_dev *dev)
46514665
{
46524666
kfree(dev->flow_db);
46534667
}
46544668

4655-
static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev)
4669+
int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev)
46564670
{
46574671
struct mlx5_core_dev *mdev = dev->mdev;
46584672
int err;
@@ -4793,7 +4807,7 @@ static int mlx5_ib_stage_non_default_cb(struct mlx5_ib_dev *dev)
47934807
return 0;
47944808
}
47954809

4796-
static int mlx5_ib_stage_rep_non_default_cb(struct mlx5_ib_dev *dev)
4810+
int mlx5_ib_stage_rep_non_default_cb(struct mlx5_ib_dev *dev)
47974811
{
47984812
dev->ib_dev.get_port_immutable = mlx5_port_rep_immutable;
47994813
dev->ib_dev.query_port = mlx5_ib_rep_query_port;
@@ -4905,12 +4919,12 @@ static void mlx5_ib_stage_roce_cleanup(struct mlx5_ib_dev *dev)
49054919
}
49064920
}
49074921

4908-
static int mlx5_ib_stage_dev_res_init(struct mlx5_ib_dev *dev)
4922+
int mlx5_ib_stage_dev_res_init(struct mlx5_ib_dev *dev)
49094923
{
49104924
return create_dev_resources(&dev->devr);
49114925
}
49124926

4913-
static void mlx5_ib_stage_dev_res_cleanup(struct mlx5_ib_dev *dev)
4927+
void mlx5_ib_stage_dev_res_cleanup(struct mlx5_ib_dev *dev)
49144928
{
49154929
destroy_dev_resources(&dev->devr);
49164930
}
@@ -4922,7 +4936,7 @@ static int mlx5_ib_stage_odp_init(struct mlx5_ib_dev *dev)
49224936
return mlx5_ib_odp_init_one(dev);
49234937
}
49244938

4925-
static int mlx5_ib_stage_counters_init(struct mlx5_ib_dev *dev)
4939+
int mlx5_ib_stage_counters_init(struct mlx5_ib_dev *dev)
49264940
{
49274941
if (MLX5_CAP_GEN(dev->mdev, max_qp_cnt)) {
49284942
dev->ib_dev.get_hw_stats = mlx5_ib_get_hw_stats;
@@ -4934,7 +4948,7 @@ static int mlx5_ib_stage_counters_init(struct mlx5_ib_dev *dev)
49344948
return 0;
49354949
}
49364950

4937-
static void mlx5_ib_stage_counters_cleanup(struct mlx5_ib_dev *dev)
4951+
void mlx5_ib_stage_counters_cleanup(struct mlx5_ib_dev *dev)
49384952
{
49394953
if (MLX5_CAP_GEN(dev->mdev, max_qp_cnt))
49404954
mlx5_ib_dealloc_counters(dev);
@@ -4965,7 +4979,7 @@ static void mlx5_ib_stage_uar_cleanup(struct mlx5_ib_dev *dev)
49654979
mlx5_put_uars_page(dev->mdev, dev->mdev->priv.uar);
49664980
}
49674981

4968-
static int mlx5_ib_stage_bfrag_init(struct mlx5_ib_dev *dev)
4982+
int mlx5_ib_stage_bfrag_init(struct mlx5_ib_dev *dev)
49694983
{
49704984
int err;
49714985

@@ -4980,28 +4994,28 @@ static int mlx5_ib_stage_bfrag_init(struct mlx5_ib_dev *dev)
49804994
return err;
49814995
}
49824996

4983-
static void mlx5_ib_stage_bfrag_cleanup(struct mlx5_ib_dev *dev)
4997+
void mlx5_ib_stage_bfrag_cleanup(struct mlx5_ib_dev *dev)
49844998
{
49854999
mlx5_free_bfreg(dev->mdev, &dev->fp_bfreg);
49865000
mlx5_free_bfreg(dev->mdev, &dev->bfreg);
49875001
}
49885002

4989-
static int mlx5_ib_stage_ib_reg_init(struct mlx5_ib_dev *dev)
5003+
int mlx5_ib_stage_ib_reg_init(struct mlx5_ib_dev *dev)
49905004
{
49915005
return ib_register_device(&dev->ib_dev, NULL);
49925006
}
49935007

4994-
static void mlx5_ib_stage_ib_reg_cleanup(struct mlx5_ib_dev *dev)
5008+
void mlx5_ib_stage_ib_reg_cleanup(struct mlx5_ib_dev *dev)
49955009
{
49965010
ib_unregister_device(&dev->ib_dev);
49975011
}
49985012

4999-
static int mlx5_ib_stage_umr_res_init(struct mlx5_ib_dev *dev)
5013+
int mlx5_ib_stage_umr_res_init(struct mlx5_ib_dev *dev)
50005014
{
50015015
return create_umr_res(dev);
50025016
}
50035017

5004-
static void mlx5_ib_stage_umr_res_cleanup(struct mlx5_ib_dev *dev)
5018+
void mlx5_ib_stage_umr_res_cleanup(struct mlx5_ib_dev *dev)
50055019
{
50065020
destroy_umrc_res(dev);
50075021
}
@@ -5018,7 +5032,7 @@ static void mlx5_ib_stage_delay_drop_cleanup(struct mlx5_ib_dev *dev)
50185032
cancel_delay_drop(dev);
50195033
}
50205034

5021-
static int mlx5_ib_stage_class_attr_init(struct mlx5_ib_dev *dev)
5035+
int mlx5_ib_stage_class_attr_init(struct mlx5_ib_dev *dev)
50225036
{
50235037
int err;
50245038
int i;
@@ -5045,9 +5059,9 @@ static void mlx5_ib_stage_rep_reg_cleanup(struct mlx5_ib_dev *dev)
50455059
mlx5_ib_unregister_vport_reps(dev);
50465060
}
50475061

5048-
static void __mlx5_ib_remove(struct mlx5_ib_dev *dev,
5049-
const struct mlx5_ib_profile *profile,
5050-
int stage)
5062+
void __mlx5_ib_remove(struct mlx5_ib_dev *dev,
5063+
const struct mlx5_ib_profile *profile,
5064+
int stage)
50515065
{
50525066
/* Number of stages to cleanup */
50535067
while (stage) {
@@ -5061,23 +5075,14 @@ static void __mlx5_ib_remove(struct mlx5_ib_dev *dev,
50615075

50625076
static void *mlx5_ib_add_slave_port(struct mlx5_core_dev *mdev, u8 port_num);
50635077

5064-
static void *__mlx5_ib_add(struct mlx5_core_dev *mdev,
5065-
const struct mlx5_ib_profile *profile)
5078+
void *__mlx5_ib_add(struct mlx5_ib_dev *dev,
5079+
const struct mlx5_ib_profile *profile)
50665080
{
5067-
struct mlx5_ib_dev *dev;
50685081
int err;
50695082
int i;
50705083

50715084
printk_once(KERN_INFO "%s", mlx5_version);
50725085

5073-
dev = (struct mlx5_ib_dev *)ib_alloc_device(sizeof(*dev));
5074-
if (!dev)
5075-
return NULL;
5076-
5077-
dev->mdev = mdev;
5078-
dev->num_ports = max(MLX5_CAP_GEN(mdev, num_ports),
5079-
MLX5_CAP_GEN(mdev, num_vhca_ports));
5080-
50815086
for (i = 0; i < MLX5_IB_STAGE_MAX; i++) {
50825087
if (profile->stage[i].init) {
50835088
err = profile->stage[i].init(dev);
@@ -5145,6 +5150,48 @@ static const struct mlx5_ib_profile pf_profile = {
51455150
NULL),
51465151
};
51475152

5153+
static const struct mlx5_ib_profile nic_rep_profile = {
5154+
STAGE_CREATE(MLX5_IB_STAGE_INIT,
5155+
mlx5_ib_stage_init_init,
5156+
mlx5_ib_stage_init_cleanup),
5157+
STAGE_CREATE(MLX5_IB_STAGE_FLOW_DB,
5158+
mlx5_ib_stage_flow_db_init,
5159+
mlx5_ib_stage_flow_db_cleanup),
5160+
STAGE_CREATE(MLX5_IB_STAGE_CAPS,
5161+
mlx5_ib_stage_caps_init,
5162+
NULL),
5163+
STAGE_CREATE(MLX5_IB_STAGE_NON_DEFAULT_CB,
5164+
mlx5_ib_stage_rep_non_default_cb,
5165+
NULL),
5166+
STAGE_CREATE(MLX5_IB_STAGE_ROCE,
5167+
mlx5_ib_stage_rep_roce_init,
5168+
mlx5_ib_stage_rep_roce_cleanup),
5169+
STAGE_CREATE(MLX5_IB_STAGE_DEVICE_RESOURCES,
5170+
mlx5_ib_stage_dev_res_init,
5171+
mlx5_ib_stage_dev_res_cleanup),
5172+
STAGE_CREATE(MLX5_IB_STAGE_COUNTERS,
5173+
mlx5_ib_stage_counters_init,
5174+
mlx5_ib_stage_counters_cleanup),
5175+
STAGE_CREATE(MLX5_IB_STAGE_UAR,
5176+
mlx5_ib_stage_uar_init,
5177+
mlx5_ib_stage_uar_cleanup),
5178+
STAGE_CREATE(MLX5_IB_STAGE_BFREG,
5179+
mlx5_ib_stage_bfrag_init,
5180+
mlx5_ib_stage_bfrag_cleanup),
5181+
STAGE_CREATE(MLX5_IB_STAGE_IB_REG,
5182+
mlx5_ib_stage_ib_reg_init,
5183+
mlx5_ib_stage_ib_reg_cleanup),
5184+
STAGE_CREATE(MLX5_IB_STAGE_UMR_RESOURCES,
5185+
mlx5_ib_stage_umr_res_init,
5186+
mlx5_ib_stage_umr_res_cleanup),
5187+
STAGE_CREATE(MLX5_IB_STAGE_CLASS_ATTR,
5188+
mlx5_ib_stage_class_attr_init,
5189+
NULL),
5190+
STAGE_CREATE(MLX5_IB_STAGE_REP_REG,
5191+
mlx5_ib_stage_rep_reg_init,
5192+
mlx5_ib_stage_rep_reg_cleanup),
5193+
};
5194+
51485195
static void *mlx5_ib_add_slave_port(struct mlx5_core_dev *mdev, u8 port_num)
51495196
{
51505197
struct mlx5_ib_multiport_info *mpi;
@@ -5190,8 +5237,11 @@ static void *mlx5_ib_add_slave_port(struct mlx5_core_dev *mdev, u8 port_num)
51905237
static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
51915238
{
51925239
enum rdma_link_layer ll;
5240+
struct mlx5_ib_dev *dev;
51935241
int port_type_cap;
51945242

5243+
printk_once(KERN_INFO "%s", mlx5_version);
5244+
51955245
port_type_cap = MLX5_CAP_GEN(mdev, port_type);
51965246
ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
51975247

@@ -5201,7 +5251,22 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
52015251
return mlx5_ib_add_slave_port(mdev, port_num);
52025252
}
52035253

5204-
return __mlx5_ib_add(mdev, &pf_profile);
5254+
dev = (struct mlx5_ib_dev *)ib_alloc_device(sizeof(*dev));
5255+
if (!dev)
5256+
return NULL;
5257+
5258+
dev->mdev = mdev;
5259+
dev->num_ports = max(MLX5_CAP_GEN(mdev, num_ports),
5260+
MLX5_CAP_GEN(mdev, num_vhca_ports));
5261+
5262+
if (MLX5_VPORT_MANAGER(mdev) &&
5263+
mlx5_ib_eswitch_mode(mdev->priv.eswitch) == SRIOV_OFFLOADS) {
5264+
dev->rep = mlx5_ib_vport_rep(mdev->priv.eswitch, 0);
5265+
5266+
return __mlx5_ib_add(dev, &nic_rep_profile);
5267+
}
5268+
5269+
return __mlx5_ib_add(dev, &pf_profile);
52055270
}
52065271

52075272
static void mlx5_ib_remove(struct mlx5_core_dev *mdev, void *context)

0 commit comments

Comments
 (0)