uvnic issues

When there is Heartbeat loss because of Link event on  EDR fabric, uvnic
recovery fails(some times).
This is due to the fact that XVE_OPER_UP state would have cleared and
xve driver has to check this once multicast group membership is retained

While running MAXq test we are running into soft crash around
skb_try_coalesce .Since xve driver allocates PAGE_SIZE it has to use
full PAGE_SIZE for truesize of skb.

xve driver doesn't allocate enough tailroom in skbs, so IP/TCP
stacks need to reallocate skb head to pull IP/TCP headers.
this .

xve allocates some resources which are not needed by uVnic
functionality.
Fix this by using cm_supported flag.

Orabug: 22862488

Reported-by: ye jin <[email protected]>
Signed-off-by: Pradeep Gopanapalli <[email protected]>
Reviewed-by: sajid zia <[email protected]>
Signed-off-by: Qing Huang <[email protected]>

Author: Pradeep Gopanapalli

drivers/infiniband/ulp/xsigo/xscore/Makefile

@@ -2,7 +2,7 @@ obj-$(CONFIG_INFINIBAND_XSCORE) := xscore.o
 xscore-y := xscore_impl.o xs_ud.o xscore_api.o xsmp.o \
 	    xscore_stats.o xscore_uadm.o
 
-ccflags-y += -DXSIGO_LOCAL_VERSION=\"6.0.r8012\"
+ccflags-y += -DXSIGO_LOCAL_VERSION=\"6.0.r8014\"
 ccflags-y += -DRDMA_PORT_LINK_LAYER_CHANGES -DHAS_SKB_ACCESS_FUNCTIONS
 ccflags-y += -DSCSI_STRUCT_CHANGES -DSCSI_TIMEOUT_CHANGES -DLLE
 ccflags-y += -DXG_FRAG_SIZE_PRESENT -DXG_FRAG_PAGE_PRESENT

drivers/infiniband/ulp/xsigo/xsvhba/Makefile

@@ -3,7 +3,7 @@ xsvhba-y := vhba_main.o vhba_xsmp.o vhba_create.o vhba_init.o vhba_delete.o \
 	    vhba_attr.o vhba_wq.o vhba_proc.o vhba_stats.o vhba_ib.o        \
 	    vhba_scsi_intf.o vhba_align.o
 
-ccflags-y += -DXSIGO_LOCAL_VERSION=\"6.0.r8008\"
+ccflags-y += -DXSIGO_LOCAL_VERSION=\"6.0.r8014\"
 ccflags-y += -DRDMA_PORT_LINK_LAYER_CHANGES -DHAS_SKB_ACCESS_FUNCTIONS
 ccflags-y += -DSCSI_STRUCT_CHANGES -DSCSI_TIMEOUT_CHANGES -DLLE
 ccflags-y += -DXG_FRAG_SIZE_PRESENT -DXG_FRAG_PAGE_PRESENT

drivers/infiniband/ulp/xsigo/xsvnic/Makefile

@@ -1,7 +1,7 @@
 obj-$(CONFIG_INFINIBAND_XSVNIC) := xsvnic.o
 xsvnic-y := xsvnic_main.o xsvnic_stats.o
 
-ccflags-y += -DXSIGO_LOCAL_VERSION=\"6.0.r8008\"
+ccflags-y += -DXSIGO_LOCAL_VERSION=\"6.0.r8014\"
 ccflags-y += -DRDMA_PORT_LINK_LAYER_CHANGES -DHAS_SKB_ACCESS_FUNCTIONS
 ccflags-y += -DSCSI_STRUCT_CHANGES -DSCSI_TIMEOUT_CHANGES -DLLE
 ccflags-y += -DXG_FRAG_SIZE_PRESENT -DXG_FRAG_PAGE_PRESENT

drivers/infiniband/ulp/xsigo/xve/Makefile

@@ -2,7 +2,7 @@ obj-$(CONFIG_INFINIBAND_XVE) := xve.o
 xve-y := xve_main.o xve_verbs.o xve_multicast.o xve_ib.o xve_tables.o \
 	 xve_ethtool.o xve_cm.o xve_stats.o
 
-ccflags-y += -DXSIGO_LOCAL_VERSION=\"6.0.r8008\"
+ccflags-y += -DXSIGO_LOCAL_VERSION=\"6.0.r8014\"
 ccflags-y += -DRDMA_PORT_LINK_LAYER_CHANGES -DHAS_SKB_ACCESS_FUNCTIONS
 ccflags-y += -DSCSI_STRUCT_CHANGES -DSCSI_TIMEOUT_CHANGES -DLLE
 ccflags-y += -DXG_FRAG_SIZE_PRESENT -DXG_FRAG_PAGE_PRESENT

drivers/infiniband/ulp/xsigo/xve/xve.h

@@ -151,7 +151,7 @@ enum xve_flush_level {
 };
 
 enum {
-	XVE_UD_HEAD_SIZE = IB_GRH_BYTES + VLAN_ETH_HLEN + XVE_EOIB_LEN + 2048,
+	XVE_UD_HEAD_SIZE = IB_GRH_BYTES + VLAN_ETH_HLEN + XVE_EOIB_LEN,
 	XVE_UD_RX_OVN_SG = 2,	/* max buffer needed for 4K mtu */
 	XVE_UD_RX_EDR_SG = 3,	/* max buffer needed for 10K mtu */
 	XVE_CM_MTU = 0x10000 - 0x20,	/* padding to align header to 16 */
@@ -223,6 +223,7 @@ enum {
 	XVE_STATE_MACHINE,
 	XVE_STATE_MACHINE_UP,
 	XVE_STATE_MACHINE_DOWN,
+	XVE_STATE_MACHINE_IBCLEAR,
 	XVE_NAPI_POLL_COUNTER,
 	XVE_SHORT_PKT_COUNTER,
 	XVE_TX_COUNTER,
@@ -279,9 +280,12 @@ enum {
 	XVE_MAC_STILL_INUSE,
 	XVE_MAC_MOVED_COUNTER,
 
+	XVE_MCAST_NOTREADY,
 	XVE_MCAST_JOIN_TASK,
 	XVE_MCAST_LEAVE_TASK,
 	XVE_MCAST_CARRIER_TASK,
+	XVE_MCAST_ATTACH,
+	XVE_MCAST_DETACH,
 
 	XVE_TX_UD_COUNTER,
 	XVE_TX_RC_COUNTER,
@@ -886,6 +890,8 @@ struct icmp6_ndp {
 	printk(level "%s: " fmt, MODULE_NAME, ##arg)
 #define XSMP_ERROR(fmt, arg...)					\
 	PRINT(KERN_ERR, "XSMP", fmt, ##arg)
+#define DRV_PRINT(fmt, arg...)                                  \
+	PRINT(KERN_INFO, "DRV", fmt, ##arg)
 #define xve_printk(level, priv, format, arg...)			\
 	printk(level "%s: " format,				\
 		((struct xve_dev_priv *) priv)->netdev->name,	\
@@ -1091,42 +1097,6 @@ static inline void xve_put_ctx(struct xve_dev_priv *priv)
 	atomic_dec(&priv->ref_cnt);
 }
 
-/* Adjust length of skb with fragments to match received data */
-static inline void skb_put_frags(struct sk_buff *skb, unsigned int hdr_space,
-				 unsigned int length, struct sk_buff *toskb)
-{
-	int i, num_frags;
-	unsigned int size;
-
-	/* put header into skb */
-	size = min(length, hdr_space);
-	skb->tail += size;
-	skb->len += size;
-	length -= size;
-
-	num_frags = skb_shinfo(skb)->nr_frags;
-	for (i = 0; i < num_frags; i++) {
-		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
-
-		if (length == 0) {
-			/* don't need this page */
-			if (toskb)
-				skb_fill_page_desc(toskb, i, skb_frag_page(frag)
-						, 0, PAGE_SIZE);
-			else
-				__free_page(skb_shinfo(skb)->frags[i].page.p);
-			--skb_shinfo(skb)->nr_frags;
-		} else {
-			size = min_t(unsigned, length, (unsigned)PAGE_SIZE);
-
-			frag->size = size;
-			skb->data_len += size;
-			skb->truesize += size;
-			skb->len += size;
-			length -= size;
-		}
-	}
-}
 
 /* functions */
 int xve_poll(struct napi_struct *napi, int budget);

drivers/infiniband/ulp/xsigo/xve/xve_cm.c

@@ -448,6 +448,44 @@ static void xve_cm_free_rx_reap_list(struct net_device *dev)
 	}
 }
 
+
+/* Adjust length of skb with fragments to match received data */
+static inline void skb_put_frags(struct sk_buff *skb, unsigned int hdr_space,
+				 unsigned int length, struct sk_buff *toskb)
+{
+	int i, num_frags;
+	unsigned int size;
+
+	/* put header into skb */
+	size = min(length, hdr_space);
+	skb->tail += size;
+	skb->len += size;
+	length -= size;
+
+	num_frags = skb_shinfo(skb)->nr_frags;
+	for (i = 0; i < num_frags; i++) {
+		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
+
+		if (length == 0) {
+			/* don't need this page */
+			if (toskb)
+				skb_fill_page_desc(toskb, i, skb_frag_page(frag)
+						, 0, PAGE_SIZE);
+			else
+				__free_page(skb_shinfo(skb)->frags[i].page.p);
+			--skb_shinfo(skb)->nr_frags;
+		} else {
+			size = min_t(unsigned, length, (unsigned)PAGE_SIZE);
+
+			frag->size = size;
+			skb->data_len += size;
+			skb->truesize += size;
+			skb->len += size;
+			length -= size;
+		}
+	}
+}
+
 void xve_cm_handle_rx_wc(struct net_device *dev, struct ib_wc *wc)
 {
 	struct xve_dev_priv *priv = netdev_priv(dev);
@@ -982,7 +1020,7 @@ static int xve_cm_tx_init(struct xve_cm_ctx *p, struct ib_sa_path_rec *pathrec)
 	int ret;
 
 	p->tx_ring = vmalloc(priv->xve_sendq_size * sizeof(*p->tx_ring));
-	if (!p->tx_ring) {
+	if (IS_ERR(p->tx_ring)) {
 		xve_warn(priv, "failed to allocate tx ring\n");
 		ret = -ENOMEM;
 		goto err_tx;
@@ -1310,8 +1348,7 @@ static void xve_cm_create_srq(struct net_device *dev, int max_sge)
 
 	priv->cm.srq = ib_create_srq(priv->pd, &srq_init_attr);
 	if (IS_ERR(priv->cm.srq)) {
-		if (PTR_ERR(priv->cm.srq) != -ENOSYS)
-			pr_warn("%s: failed to allocate SRQ, error %ld\n",
+		pr_warn("%s: failed to allocate SRQ, error %ld\n",
 				priv->ca->name, PTR_ERR(priv->cm.srq));
 		priv->cm.srq = NULL;
 		return;
@@ -1337,6 +1374,9 @@ int xve_cm_dev_init(struct net_device *dev)
 	int i, ret;
 	struct ib_device_attr attr;
 
+	if (!priv->cm_supported)
+		return 0;
+
 	INIT_LIST_HEAD(&priv->cm.passive_ids);
 	INIT_LIST_HEAD(&priv->cm.reap_list);
 	INIT_LIST_HEAD(&priv->cm.start_list);
@@ -1351,8 +1391,6 @@ int xve_cm_dev_init(struct net_device *dev)
 		return ret;
 	}
 
-	priv->dev_attr = attr;
-
 	/* Based on the admin mtu from the chassis */
 	attr.max_srq_sge =
 	    min_t(int,
@@ -1407,7 +1445,7 @@ void xve_cm_dev_cleanup(struct net_device *dev)
 	struct xve_dev_priv *priv = netdev_priv(dev);
 	int ret;
 
-	if (!priv->cm.srq)
+	if (!priv->cm_supported || !priv->cm.srq)
 		return;
 
 	xve_debug(DEBUG_CM_INFO, priv, "%s Cleanup xve CM\n", __func__);

drivers/infiniband/ulp/xsigo/xve/xve_ethtool.c

@@ -80,7 +80,7 @@ static int xve_set_coalesce(struct net_device *dev,
 	ret = ib_modify_cq(priv->recv_cq, coal->rx_max_coalesced_frames,
 			   coal->rx_coalesce_usecs);
 
-	if (ret && ret != -ENOSYS) {
+	if (ret) {
 		xve_warn(priv, "failed modifying CQ (%d)\n", ret);
 		return ret;
 	}

drivers/infiniband/ulp/xsigo/xve/xve_ib.c

@@ -87,6 +87,30 @@ static void xve_ud_dma_unmap_rx(struct xve_dev_priv *priv,
 	}
 }
 
+
+static void xve_ud_skb_put_frags(struct xve_dev_priv *priv,
+		struct sk_buff *skb,
+		unsigned int length)
+{
+	if (xve_ud_need_sg(priv->max_ib_mtu)) {
+		skb_frag_t *frag = &skb_shinfo(skb)->frags[0];
+		unsigned int size;
+		/*
+		 * There is only two buffers needed for max_payload = 4K,
+		 * first buf size is XVE_UD_HEAD_SIZE
+		 */
+		skb->tail += XVE_UD_HEAD_SIZE;
+		skb->len  += length;
+
+		size = length - XVE_UD_HEAD_SIZE;
+
+		skb_frag_size_set(frag, size);
+		skb->data_len += size;
+		skb->truesize += PAGE_SIZE;
+	} else
+		skb_put(skb, length);
+}
+
 static int xve_ib_post_receive(struct net_device *dev, int id)
 {
 	struct xve_dev_priv *priv = netdev_priv(dev);
@@ -114,11 +138,16 @@ static struct sk_buff *xve_alloc_rx_skb(struct net_device *dev, int id)
 	struct sk_buff *skb;
 	int buf_size, align;
 	u64 *mapping;
+	int tailroom;
 
-	if (xve_ud_need_sg(priv->max_ib_mtu))
+	if (xve_ud_need_sg(priv->max_ib_mtu)) {
+		/* reserve some tailroom for IP/TCP headers */
 		buf_size = XVE_UD_HEAD_SIZE;
-	else
+		tailroom = 128;
+	} else {
 		buf_size = XVE_UD_BUF_SIZE(priv->max_ib_mtu);
+		tailroom = 0;
+	}
 
 	/*
 	 * Eth header is 14 bytes, IB will leave a 40 byte gap for a GRH
@@ -127,7 +156,7 @@ static struct sk_buff *xve_alloc_rx_skb(struct net_device *dev, int id)
 	 * 4-byte EoIB header.
 	 */
 	align = xve_is_ovn(priv) ? 10 : 6;
-	skb = xve_dev_alloc_skb(priv, buf_size + align);
+	skb = xve_dev_alloc_skb(priv, buf_size + tailroom + align);
 	if (unlikely(!skb))
 		return NULL;
 
@@ -207,7 +236,8 @@ void xve_process_link_state(struct xve_dev_priv *priv,
 		set_bit(XVE_GW_STATE_UP, &priv->state);
 		priv->hb_interval = 30*HZ;
 
-		if (!netif_carrier_ok(priv->netdev))
+		if ((!netif_carrier_ok(priv->netdev)) ||
+			(!test_bit(XVE_OPER_UP, &priv->flags)))
 			xve_link_up(priv);
 	} else {
 		clear_bit(XVE_GW_STATE_UP, &priv->state);
@@ -323,7 +353,7 @@ xve_ib_handle_rx_wc(struct net_device *dev, struct ib_wc *wc)
 		     wc->byte_len, wc->slid);
 
 	xve_ud_dma_unmap_rx(priv, mapping);
-	skb_put_frags(skb, XVE_UD_HEAD_SIZE, wc->byte_len, NULL);
+	xve_ud_skb_put_frags(priv, skb,  wc->byte_len);
 	grhhdr = (struct ib_packed_grh *)(skb->data);
 	/* This will print packet when driver is in Debug Mode */
 	dumppkt(skb->data, skb->len, "UD Packet Dump");
@@ -507,18 +537,14 @@ int poll_tx(struct xve_dev_priv *priv)
 	do {
 		n = ib_poll_cq(priv->send_cq, MAX_SEND_CQE, priv->send_wc);
 		/* handle multiple WC's in one call */
-		if (likely(n > 0)) {
-			for (i = 0; i < n; ++i)
-				xve_ib_handle_tx_wc(priv->netdev,
-						    priv->send_wc + i);
-			tot += n;
-		} else if (n == 0) {
-			break;
-		} else {
+		for (i = 0; i < n; ++i)
+			xve_ib_handle_tx_wc(priv->netdev,
+					priv->send_wc + i);
+		if (n < 0) {
 			xve_warn(priv, "%s ib_poll_cq() failed, rc %d\n",
-				 __func__, n);
+					__func__, n);
 		}
-
+		tot += n;
 	} while (n == MAX_SEND_CQE);
 
 	return tot;

drivers/infiniband/ulp/xsigo/xve/xve_main.c

@@ -1077,14 +1077,14 @@ int xve_dev_init(struct net_device *dev, struct ib_device *ca, int port)
 	priv->rx_ring = kcalloc(priv->xve_recvq_size, sizeof(*priv->rx_ring),
 				GFP_KERNEL);
 	if (!priv->rx_ring) {
-		pr_warn("%s: failed to allocate RX ring (%d entries)\n",
+		DRV_PRINT("%s:failed to allocate RX ring (%d entries)\n",
 			ca->name, priv->xve_recvq_size);
 		goto out;
 	}
 
 	priv->tx_ring = vmalloc(priv->xve_sendq_size * sizeof(*priv->tx_ring));
 	if (!priv->tx_ring) {
-		pr_warn("%s: failed to allocate TX ring (%d entries)\n",
+		DRV_PRINT("%s:failed to allocate TX ring (%d entries)\n",
 			ca->name, priv->xve_sendq_size);
 		goto out_rx_ring_cleanup;
 	}
@@ -1761,7 +1761,7 @@ xve_set_edr_features(struct xve_dev_priv *priv)
 
 int xve_set_dev_features(struct xve_dev_priv *priv, struct ib_device *hca)
 {
-	struct ib_device_attr *device_attr;
+	struct ib_device_attr device_attr;
 	int result = -ENOMEM;
 
 	priv->netdev->watchdog_timeo = 1000 * HZ;
@@ -1796,23 +1796,15 @@ int xve_set_dev_features(struct xve_dev_priv *priv, struct ib_device *hca)
 
 	xve_set_netdev(priv->netdev);
 
-	device_attr = kmalloc(sizeof(*device_attr), GFP_KERNEL);
-
-	if (!device_attr) {
-		pr_warn("%s: allocation of %zu bytes failed\n",
-				hca->name, sizeof(*device_attr));
-		return result;
-	}
-
-	result = ib_query_device(hca, device_attr);
+	result = ib_query_device(hca, &device_attr);
 	if (result) {
 		pr_warn("%s: ib_query_device failed (ret = %d)\n",
 				hca->name, result);
-		kfree(device_attr);
 		return result;
 	}
-	priv->hca_caps = device_attr->device_cap_flags;
-	kfree(device_attr);
+
+	priv->dev_attr = device_attr;
+	priv->hca_caps = device_attr.device_cap_flags;
 
 	xve_lro_setup(priv);
 	if (xve_is_ovn(priv))
@@ -1969,8 +1961,8 @@ int xve_xsmp_send_oper_state(struct xve_dev_priv *priv, u64 vid, int state)
 	int ret;
 	char *str = state == XSMP_XVE_OPER_UP ? "UP" : "DOWN";
 
-	pr_info("XVE: %s Sending OPER state [%d]  to %s\n",
-		__func__, state, priv->xve_name);
+	pr_info("XVE: %s Sending OPER state [%d:%s]  to %s\n",
+		__func__, state, str, priv->xve_name);
 	if (state == XSMP_XVE_OPER_UP) {
 		set_bit(XVE_OPER_REP_SENT, &priv->state);
 		set_bit(XVE_PORT_LINK_UP, &priv->state);