net/rds: rds_ib_remove_one() needs to wait

The RDMA device removal upcall (for RDS, it is rds_ib_remove_one())
expects that all resources related to the device should be cleaned up
before the upcall returns.  After the upcall returns, it will proceed
to clean up the device related resources.  This means that
rds_ib_remove_one() has to wait for all the related RDS resources to
be cleaned up before returning.

Orabug: 31794611

Signed-off-by: Ka-Cheong Poon <[email protected]>
Tested-by: Suhua Ding <[email protected]>
Tested-by: Jenny Xu <[email protected]>
Reviewed-by: Håkon Bugge <[email protected]>
Signed-off-by: Somasundaram Krishnasamy <[email protected]>

Author: kcp-git

net/rds/ib.c

@@ -426,6 +426,7 @@ static void rds_ib_dev_free(struct work_struct *work)
 	if (rds_ibdev->vector_load)
 		kfree(rds_ibdev->vector_load);
 
+	complete(rds_ibdev->rid_dev_rem_complete);
 	kfree(rds_ibdev);
 }
 
@@ -473,6 +474,7 @@ struct rds_ib_device *rds_ib_get_client_data(struct ib_device *device)
  */
 void rds_ib_remove_one(struct ib_device *device, void *client_data)
 {
+	DECLARE_COMPLETION(rem_complete);
 	struct rds_ib_device *rds_ibdev;
 
 	rds_rtd(RDS_RTD_RDMA_IB, "Removing ib_device: %p name: %s num_ports: %u\n",
@@ -499,6 +501,8 @@ void rds_ib_remove_one(struct ib_device *device, void *client_data)
 	list_del_rcu(&rds_ibdev->list);
 	up_write(&rds_ib_devices_lock);
 
+	rds_ibdev->rid_dev_rem_complete = &rem_complete;
+
 	/*
 	 * This synchronize rcu is waiting for readers of both the ib
 	 * client data and the devices list to finish before we drop
@@ -507,6 +511,7 @@ void rds_ib_remove_one(struct ib_device *device, void *client_data)
 	synchronize_rcu();
 	rds_ib_dev_put(rds_ibdev);
 	rds_ib_dev_put(rds_ibdev);
+	wait_for_completion(&rem_complete);
 }
 
 struct ib_client rds_ib_client = {

net/rds/ib.h

@@ -385,6 +385,8 @@ struct rds_ib_device {
 	struct rds_ib_refill_cache i_cache_incs;
 	struct rds_ib_refill_cache i_cache_frags[RDS_FRAG_CACHE_ENTRIES];
 	struct dentry *debugfs_dir;
+
+	struct completion	*rid_dev_rem_complete;
 };
 
 static inline int ibdev_to_node(struct ib_device *ibdev)