Skip to content

Commit f8a70af

Browse files
rleonklassert
rleon
authored and
klassert
committed
xfrm: add TX datapath support for IPsec packet offload mode
In IPsec packet mode, the device is going to encrypt and encapsulate packets that are associated with offloaded policy. After successful policy lookup to indicate if packets should be offloaded or not, the stack forwards packets to the device to do the magic. Signed-off-by: Raed Salem <[email protected]> Signed-off-by: Huy Nguyen <[email protected]> Signed-off-by: Leon Romanovsky <[email protected]> Signed-off-by: Steffen Klassert <[email protected]>
1 parent 919e43f commit f8a70af

File tree

3 files changed

+142
-6
lines changed

3 files changed

+142
-6
lines changed

‎net/xfrm/xfrm_device.c

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,16 @@ struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t featur
132132
if (xo->flags & XFRM_GRO || x->xso.dir == XFRM_DEV_OFFLOAD_IN)
133133
return skb;
134134

135+
/* The packet was sent to HW IPsec packet offload engine,
136+
* but to wrong device. Drop the packet, so it won't skip
137+
* XFRM stack.
138+
*/
139+
if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET && x->xso.dev != dev) {
140+
kfree_skb(skb);
141+
dev_core_stats_tx_dropped_inc(dev);
142+
return NULL;
143+
}
144+
135145
/* This skb was already validated on the upper/virtual dev */
136146
if ((x->xso.dev != dev) && (x->xso.real_dev == dev))
137147
return skb;
@@ -398,8 +408,9 @@ bool xfrm_dev_offload_ok(struct sk_buff *skb, struct xfrm_state *x)
398408
if (!x->type_offload || x->encap)
399409
return false;
400410

401-
if ((!dev || (dev == xfrm_dst_path(dst)->dev)) &&
402-
(!xdst->child->xfrm)) {
411+
if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET ||
412+
((!dev || (dev == xfrm_dst_path(dst)->dev)) &&
413+
!xdst->child->xfrm)) {
403414
mtu = xfrm_state_mtu(x, xdst->child_mtu_cached);
404415
if (skb->len <= mtu)
405416
goto ok;

‎net/xfrm/xfrm_output.c

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -492,7 +492,7 @@ static int xfrm_output_one(struct sk_buff *skb, int err)
492492
struct xfrm_state *x = dst->xfrm;
493493
struct net *net = xs_net(x);
494494

495-
if (err <= 0)
495+
if (err <= 0 || x->xso.type == XFRM_DEV_OFFLOAD_PACKET)
496496
goto resume;
497497

498498
do {
@@ -717,6 +717,16 @@ int xfrm_output(struct sock *sk, struct sk_buff *skb)
717717
break;
718718
}
719719

720+
if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET) {
721+
if (!xfrm_dev_offload_ok(skb, x)) {
722+
XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR);
723+
kfree_skb(skb);
724+
return -EHOSTUNREACH;
725+
}
726+
727+
return xfrm_output_resume(sk, skb, 0);
728+
}
729+
720730
secpath_reset(skb);
721731

722732
if (xfrm_dev_offload_ok(skb, x)) {

‎net/xfrm/xfrm_state.c

Lines changed: 118 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -951,6 +951,49 @@ xfrm_init_tempstate(struct xfrm_state *x, const struct flowi *fl,
951951
x->props.family = tmpl->encap_family;
952952
}
953953

954+
static struct xfrm_state *__xfrm_state_lookup_all(struct net *net, u32 mark,
955+
const xfrm_address_t *daddr,
956+
__be32 spi, u8 proto,
957+
unsigned short family,
958+
struct xfrm_dev_offload *xdo)
959+
{
960+
unsigned int h = xfrm_spi_hash(net, daddr, spi, proto, family);
961+
struct xfrm_state *x;
962+
963+
hlist_for_each_entry_rcu(x, net->xfrm.state_byspi + h, byspi) {
964+
#ifdef CONFIG_XFRM_OFFLOAD
965+
if (xdo->type == XFRM_DEV_OFFLOAD_PACKET) {
966+
if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET)
967+
/* HW states are in the head of list, there is
968+
* no need to iterate further.
969+
*/
970+
break;
971+
972+
/* Packet offload: both policy and SA should
973+
* have same device.
974+
*/
975+
if (xdo->dev != x->xso.dev)
976+
continue;
977+
} else if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET)
978+
/* Skip HW policy for SW lookups */
979+
continue;
980+
#endif
981+
if (x->props.family != family ||
982+
x->id.spi != spi ||
983+
x->id.proto != proto ||
984+
!xfrm_addr_equal(&x->id.daddr, daddr, family))
985+
continue;
986+
987+
if ((mark & x->mark.m) != x->mark.v)
988+
continue;
989+
if (!xfrm_state_hold_rcu(x))
990+
continue;
991+
return x;
992+
}
993+
994+
return NULL;
995+
}
996+
954997
static struct xfrm_state *__xfrm_state_lookup(struct net *net, u32 mark,
955998
const xfrm_address_t *daddr,
956999
__be32 spi, u8 proto,
@@ -1092,6 +1135,23 @@ xfrm_state_find(const xfrm_address_t *daddr, const xfrm_address_t *saddr,
10921135
rcu_read_lock();
10931136
h = xfrm_dst_hash(net, daddr, saddr, tmpl->reqid, encap_family);
10941137
hlist_for_each_entry_rcu(x, net->xfrm.state_bydst + h, bydst) {
1138+
#ifdef CONFIG_XFRM_OFFLOAD
1139+
if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) {
1140+
if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET)
1141+
/* HW states are in the head of list, there is
1142+
* no need to iterate further.
1143+
*/
1144+
break;
1145+
1146+
/* Packet offload: both policy and SA should
1147+
* have same device.
1148+
*/
1149+
if (pol->xdo.dev != x->xso.dev)
1150+
continue;
1151+
} else if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET)
1152+
/* Skip HW policy for SW lookups */
1153+
continue;
1154+
#endif
10951155
if (x->props.family == encap_family &&
10961156
x->props.reqid == tmpl->reqid &&
10971157
(mark & x->mark.m) == x->mark.v &&
@@ -1109,6 +1169,23 @@ xfrm_state_find(const xfrm_address_t *daddr, const xfrm_address_t *saddr,
11091169

11101170
h_wildcard = xfrm_dst_hash(net, daddr, &saddr_wildcard, tmpl->reqid, encap_family);
11111171
hlist_for_each_entry_rcu(x, net->xfrm.state_bydst + h_wildcard, bydst) {
1172+
#ifdef CONFIG_XFRM_OFFLOAD
1173+
if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) {
1174+
if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET)
1175+
/* HW states are in the head of list, there is
1176+
* no need to iterate further.
1177+
*/
1178+
break;
1179+
1180+
/* Packet offload: both policy and SA should
1181+
* have same device.
1182+
*/
1183+
if (pol->xdo.dev != x->xso.dev)
1184+
continue;
1185+
} else if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET)
1186+
/* Skip HW policy for SW lookups */
1187+
continue;
1188+
#endif
11121189
if (x->props.family == encap_family &&
11131190
x->props.reqid == tmpl->reqid &&
11141191
(mark & x->mark.m) == x->mark.v &&
@@ -1126,8 +1203,10 @@ xfrm_state_find(const xfrm_address_t *daddr, const xfrm_address_t *saddr,
11261203
x = best;
11271204
if (!x && !error && !acquire_in_progress) {
11281205
if (tmpl->id.spi &&
1129-
(x0 = __xfrm_state_lookup(net, mark, daddr, tmpl->id.spi,
1130-
tmpl->id.proto, encap_family)) != NULL) {
1206+
(x0 = __xfrm_state_lookup_all(net, mark, daddr,
1207+
tmpl->id.spi, tmpl->id.proto,
1208+
encap_family,
1209+
&pol->xdo)) != NULL) {
11311210
to_put = x0;
11321211
error = -EEXIST;
11331212
goto out;
@@ -1161,7 +1240,31 @@ xfrm_state_find(const xfrm_address_t *daddr, const xfrm_address_t *saddr,
11611240
x = NULL;
11621241
goto out;
11631242
}
1164-
1243+
#ifdef CONFIG_XFRM_OFFLOAD
1244+
if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) {
1245+
struct xfrm_dev_offload *xdo = &pol->xdo;
1246+
struct xfrm_dev_offload *xso = &x->xso;
1247+
1248+
xso->type = XFRM_DEV_OFFLOAD_PACKET;
1249+
xso->dir = xdo->dir;
1250+
xso->dev = xdo->dev;
1251+
xso->real_dev = xdo->real_dev;
1252+
netdev_tracker_alloc(xso->dev, &xso->dev_tracker,
1253+
GFP_ATOMIC);
1254+
error = xso->dev->xfrmdev_ops->xdo_dev_state_add(x);
1255+
if (error) {
1256+
xso->dir = 0;
1257+
netdev_put(xso->dev, &xso->dev_tracker);
1258+
xso->dev = NULL;
1259+
xso->real_dev = NULL;
1260+
xso->type = XFRM_DEV_OFFLOAD_UNSPECIFIED;
1261+
x->km.state = XFRM_STATE_DEAD;
1262+
to_put = x;
1263+
x = NULL;
1264+
goto out;
1265+
}
1266+
}
1267+
#endif
11651268
if (km_query(x, tmpl, pol) == 0) {
11661269
spin_lock_bh(&net->xfrm.xfrm_state_lock);
11671270
x->km.state = XFRM_STATE_ACQ;
@@ -1185,6 +1288,18 @@ xfrm_state_find(const xfrm_address_t *daddr, const xfrm_address_t *saddr,
11851288
xfrm_hash_grow_check(net, x->bydst.next != NULL);
11861289
spin_unlock_bh(&net->xfrm.xfrm_state_lock);
11871290
} else {
1291+
#ifdef CONFIG_XFRM_OFFLOAD
1292+
struct xfrm_dev_offload *xso = &x->xso;
1293+
1294+
if (xso->type == XFRM_DEV_OFFLOAD_PACKET) {
1295+
xso->dev->xfrmdev_ops->xdo_dev_state_delete(x);
1296+
xso->dir = 0;
1297+
netdev_put(xso->dev, &xso->dev_tracker);
1298+
xso->dev = NULL;
1299+
xso->real_dev = NULL;
1300+
xso->type = XFRM_DEV_OFFLOAD_UNSPECIFIED;
1301+
}
1302+
#endif
11881303
x->km.state = XFRM_STATE_DEAD;
11891304
to_put = x;
11901305
x = NULL;

0 commit comments

Comments
 (0)