Skip to content

Commit 737a234

Browse files
Mohamad Haj Yahiadavem330
Mohamad Haj Yahia
authored and
davem330
committed
net/mlx5: Introduce attach/detach to interface API
Add attach/detach callbacks to interface API. This is crucial for implementing seamless reset flow which releases the hardware and it's resources upon detach while keeping software structures and state (e.g netdev) then reset and reallocate the hardware needed resources upon attach. Signed-off-by: Mohamad Haj Yahia <[email protected]> Signed-off-by: Saeed Mahameed <[email protected]> Signed-off-by: David S. Miller <[email protected]>
1 parent 6b6adee commit 737a234

File tree

2 files changed

+131
-20
lines changed
  • drivers/net/ethernet/mellanox/mlx5/core
  • include/linux/mlx5

2 files changed

+131
-20
lines changed

‎drivers/net/ethernet/mellanox/mlx5/core/main.c

Lines changed: 129 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ struct mlx5_device_context {
8181
struct list_head list;
8282
struct mlx5_interface *intf;
8383
void *context;
84+
unsigned long state;
8485
};
8586

8687
enum {
@@ -778,6 +779,11 @@ static int mlx5_core_set_issi(struct mlx5_core_dev *dev)
778779
return -ENOTSUPP;
779780
}
780781

782+
enum {
783+
MLX5_INTERFACE_ADDED,
784+
MLX5_INTERFACE_ATTACHED,
785+
};
786+
781787
static void mlx5_add_device(struct mlx5_interface *intf, struct mlx5_priv *priv)
782788
{
783789
struct mlx5_device_context *dev_ctx;
@@ -786,12 +792,15 @@ static void mlx5_add_device(struct mlx5_interface *intf, struct mlx5_priv *priv)
786792
if (!mlx5_lag_intf_add(intf, priv))
787793
return;
788794

789-
dev_ctx = kmalloc(sizeof(*dev_ctx), GFP_KERNEL);
795+
dev_ctx = kzalloc(sizeof(*dev_ctx), GFP_KERNEL);
790796
if (!dev_ctx)
791797
return;
792798

793-
dev_ctx->intf = intf;
799+
dev_ctx->intf = intf;
794800
dev_ctx->context = intf->add(dev);
801+
set_bit(MLX5_INTERFACE_ADDED, &dev_ctx->state);
802+
if (intf->attach)
803+
set_bit(MLX5_INTERFACE_ATTACHED, &dev_ctx->state);
795804

796805
if (dev_ctx->context) {
797806
spin_lock_irq(&priv->ctx_lock);
@@ -802,21 +811,114 @@ static void mlx5_add_device(struct mlx5_interface *intf, struct mlx5_priv *priv)
802811
}
803812
}
804813

814+
static struct mlx5_device_context *mlx5_get_device(struct mlx5_interface *intf,
815+
struct mlx5_priv *priv)
816+
{
817+
struct mlx5_device_context *dev_ctx;
818+
819+
list_for_each_entry(dev_ctx, &priv->ctx_list, list)
820+
if (dev_ctx->intf == intf)
821+
return dev_ctx;
822+
return NULL;
823+
}
824+
805825
static void mlx5_remove_device(struct mlx5_interface *intf, struct mlx5_priv *priv)
806826
{
807827
struct mlx5_device_context *dev_ctx;
808828
struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev, priv);
809829

810-
list_for_each_entry(dev_ctx, &priv->ctx_list, list)
811-
if (dev_ctx->intf == intf) {
812-
spin_lock_irq(&priv->ctx_lock);
813-
list_del(&dev_ctx->list);
814-
spin_unlock_irq(&priv->ctx_lock);
830+
dev_ctx = mlx5_get_device(intf, priv);
831+
if (!dev_ctx)
832+
return;
833+
834+
spin_lock_irq(&priv->ctx_lock);
835+
list_del(&dev_ctx->list);
836+
spin_unlock_irq(&priv->ctx_lock);
837+
838+
if (test_bit(MLX5_INTERFACE_ADDED, &dev_ctx->state))
839+
intf->remove(dev, dev_ctx->context);
815840

816-
intf->remove(dev, dev_ctx->context);
817-
kfree(dev_ctx);
841+
kfree(dev_ctx);
842+
}
843+
844+
static void mlx5_attach_interface(struct mlx5_interface *intf, struct mlx5_priv *priv)
845+
{
846+
struct mlx5_device_context *dev_ctx;
847+
struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev, priv);
848+
849+
dev_ctx = mlx5_get_device(intf, priv);
850+
if (!dev_ctx)
851+
return;
852+
853+
if (intf->attach) {
854+
if (test_bit(MLX5_INTERFACE_ATTACHED, &dev_ctx->state))
818855
return;
819-
}
856+
intf->attach(dev, dev_ctx->context);
857+
set_bit(MLX5_INTERFACE_ATTACHED, &dev_ctx->state);
858+
} else {
859+
if (test_bit(MLX5_INTERFACE_ADDED, &dev_ctx->state))
860+
return;
861+
dev_ctx->context = intf->add(dev);
862+
set_bit(MLX5_INTERFACE_ADDED, &dev_ctx->state);
863+
}
864+
}
865+
866+
static void mlx5_attach_device(struct mlx5_core_dev *dev)
867+
{
868+
struct mlx5_priv *priv = &dev->priv;
869+
struct mlx5_interface *intf;
870+
871+
mutex_lock(&mlx5_intf_mutex);
872+
list_for_each_entry(intf, &intf_list, list)
873+
mlx5_attach_interface(intf, priv);
874+
mutex_unlock(&mlx5_intf_mutex);
875+
}
876+
877+
static void mlx5_detach_interface(struct mlx5_interface *intf, struct mlx5_priv *priv)
878+
{
879+
struct mlx5_device_context *dev_ctx;
880+
struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev, priv);
881+
882+
dev_ctx = mlx5_get_device(intf, priv);
883+
if (!dev_ctx)
884+
return;
885+
886+
if (intf->detach) {
887+
if (!test_bit(MLX5_INTERFACE_ATTACHED, &dev_ctx->state))
888+
return;
889+
intf->detach(dev, dev_ctx->context);
890+
clear_bit(MLX5_INTERFACE_ATTACHED, &dev_ctx->state);
891+
} else {
892+
if (!test_bit(MLX5_INTERFACE_ADDED, &dev_ctx->state))
893+
return;
894+
intf->remove(dev, dev_ctx->context);
895+
clear_bit(MLX5_INTERFACE_ADDED, &dev_ctx->state);
896+
}
897+
}
898+
899+
static void mlx5_detach_device(struct mlx5_core_dev *dev)
900+
{
901+
struct mlx5_priv *priv = &dev->priv;
902+
struct mlx5_interface *intf;
903+
904+
mutex_lock(&mlx5_intf_mutex);
905+
list_for_each_entry(intf, &intf_list, list)
906+
mlx5_detach_interface(intf, priv);
907+
mutex_unlock(&mlx5_intf_mutex);
908+
}
909+
910+
static bool mlx5_device_registered(struct mlx5_core_dev *dev)
911+
{
912+
struct mlx5_priv *priv;
913+
bool found = false;
914+
915+
mutex_lock(&mlx5_intf_mutex);
916+
list_for_each_entry(priv, &mlx5_dev_list, dev_list)
917+
if (priv == &dev->priv)
918+
found = true;
919+
mutex_unlock(&mlx5_intf_mutex);
920+
921+
return found;
820922
}
821923

822924
static int mlx5_register_device(struct mlx5_core_dev *dev)
@@ -1162,16 +1264,16 @@ static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
11621264
goto err_sriov;
11631265
}
11641266

1165-
err = mlx5_register_device(dev);
1166-
if (err) {
1167-
dev_err(&pdev->dev, "mlx5_register_device failed %d\n", err);
1168-
goto err_reg_dev;
1267+
if (mlx5_device_registered(dev)) {
1268+
mlx5_attach_device(dev);
1269+
} else {
1270+
err = mlx5_register_device(dev);
1271+
if (err) {
1272+
dev_err(&pdev->dev, "mlx5_register_device failed %d\n", err);
1273+
goto err_reg_dev;
1274+
}
11691275
}
11701276

1171-
err = request_module_nowait(MLX5_IB_MOD);
1172-
if (err)
1173-
pr_info("failed request module on %s\n", MLX5_IB_MOD);
1174-
11751277
clear_bit(MLX5_INTERFACE_STATE_DOWN, &dev->intf_state);
11761278
set_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state);
11771279
out:
@@ -1247,12 +1349,13 @@ static int mlx5_unload_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
12471349
goto out;
12481350
}
12491351

1352+
if (mlx5_device_registered(dev))
1353+
mlx5_detach_device(dev);
1354+
12501355
mlx5_sriov_cleanup(dev);
1251-
mlx5_unregister_device(dev);
12521356
#ifdef CONFIG_MLX5_CORE_EN
12531357
mlx5_eswitch_cleanup(dev->priv.eswitch);
12541358
#endif
1255-
12561359
mlx5_cleanup_rl_table(dev);
12571360
mlx5_cleanup_fs(dev);
12581361
mlx5_cleanup_mkey_table(dev);
@@ -1364,6 +1467,9 @@ static int init_one(struct pci_dev *pdev,
13641467
dev_err(&pdev->dev, "mlx5_load_one failed with error code %d\n", err);
13651468
goto clean_health;
13661469
}
1470+
err = request_module_nowait(MLX5_IB_MOD);
1471+
if (err)
1472+
pr_info("failed request module on %s\n", MLX5_IB_MOD);
13671473

13681474
err = devlink_register(devlink, &pdev->dev);
13691475
if (err)
@@ -1391,11 +1497,14 @@ static void remove_one(struct pci_dev *pdev)
13911497
struct mlx5_priv *priv = &dev->priv;
13921498

13931499
devlink_unregister(devlink);
1500+
mlx5_unregister_device(dev);
1501+
13941502
if (mlx5_unload_one(dev, priv)) {
13951503
dev_err(&dev->pdev->dev, "mlx5_unload_one failed\n");
13961504
mlx5_health_cleanup(dev);
13971505
return;
13981506
}
1507+
13991508
mlx5_health_cleanup(dev);
14001509
mlx5_pci_close(dev, priv);
14011510
pci_set_drvdata(pdev, NULL);

‎include/linux/mlx5/driver.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -930,6 +930,8 @@ enum {
930930
struct mlx5_interface {
931931
void * (*add)(struct mlx5_core_dev *dev);
932932
void (*remove)(struct mlx5_core_dev *dev, void *context);
933+
int (*attach)(struct mlx5_core_dev *dev, void *context);
934+
void (*detach)(struct mlx5_core_dev *dev, void *context);
933935
void (*event)(struct mlx5_core_dev *dev, void *context,
934936
enum mlx5_dev_event event, unsigned long param);
935937
void * (*get_dev)(void *context);

0 commit comments

Comments
 (0)