Skip to content

Commit 7f0e810

Browse files
tammeladavem330
tammela
authored and
davem330
committed
net/sched: add retpoline wrapper for tc
On kernels using retpoline as a spectrev2 mitigation, optimize actions and filters that are compiled as built-ins into a direct call. On subsequent patches we expose the classifiers and actions functions and wire up the wrapper into tc. Signed-off-by: Pedro Tammela <[email protected]> Reviewed-by: Jamal Hadi Salim <[email protected]> Reviewed-by: Victor Nogueira <[email protected]> Signed-off-by: David S. Miller <[email protected]>
1 parent 2a7d228 commit 7f0e810

File tree

2 files changed

+256
-0
lines changed

2 files changed

+256
-0
lines changed

‎include/net/tc_wrapper.h

Lines changed: 251 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,251 @@
1+
/* SPDX-License-Identifier: GPL-2.0 */
2+
#ifndef __NET_TC_WRAPPER_H
3+
#define __NET_TC_WRAPPER_H
4+
5+
#include <net/pkt_cls.h>
6+
7+
#if IS_ENABLED(CONFIG_RETPOLINE)
8+
9+
#include <linux/cpufeature.h>
10+
#include <linux/static_key.h>
11+
#include <linux/indirect_call_wrapper.h>
12+
13+
#define TC_INDIRECT_SCOPE
14+
15+
extern struct static_key_false tc_skip_wrapper;
16+
17+
/* TC Actions */
18+
#ifdef CONFIG_NET_CLS_ACT
19+
20+
#define TC_INDIRECT_ACTION_DECLARE(fname) \
21+
INDIRECT_CALLABLE_DECLARE(int fname(struct sk_buff *skb, \
22+
const struct tc_action *a, \
23+
struct tcf_result *res))
24+
25+
TC_INDIRECT_ACTION_DECLARE(tcf_bpf_act);
26+
TC_INDIRECT_ACTION_DECLARE(tcf_connmark_act);
27+
TC_INDIRECT_ACTION_DECLARE(tcf_csum_act);
28+
TC_INDIRECT_ACTION_DECLARE(tcf_ct_act);
29+
TC_INDIRECT_ACTION_DECLARE(tcf_ctinfo_act);
30+
TC_INDIRECT_ACTION_DECLARE(tcf_gact_act);
31+
TC_INDIRECT_ACTION_DECLARE(tcf_gate_act);
32+
TC_INDIRECT_ACTION_DECLARE(tcf_ife_act);
33+
TC_INDIRECT_ACTION_DECLARE(tcf_ipt_act);
34+
TC_INDIRECT_ACTION_DECLARE(tcf_mirred_act);
35+
TC_INDIRECT_ACTION_DECLARE(tcf_mpls_act);
36+
TC_INDIRECT_ACTION_DECLARE(tcf_nat_act);
37+
TC_INDIRECT_ACTION_DECLARE(tcf_pedit_act);
38+
TC_INDIRECT_ACTION_DECLARE(tcf_police_act);
39+
TC_INDIRECT_ACTION_DECLARE(tcf_sample_act);
40+
TC_INDIRECT_ACTION_DECLARE(tcf_simp_act);
41+
TC_INDIRECT_ACTION_DECLARE(tcf_skbedit_act);
42+
TC_INDIRECT_ACTION_DECLARE(tcf_skbmod_act);
43+
TC_INDIRECT_ACTION_DECLARE(tcf_vlan_act);
44+
TC_INDIRECT_ACTION_DECLARE(tunnel_key_act);
45+
46+
static inline int tc_act(struct sk_buff *skb, const struct tc_action *a,
47+
struct tcf_result *res)
48+
{
49+
if (static_branch_likely(&tc_skip_wrapper))
50+
goto skip;
51+
52+
#if IS_BUILTIN(CONFIG_NET_ACT_GACT)
53+
if (a->ops->act == tcf_gact_act)
54+
return tcf_gact_act(skb, a, res);
55+
#endif
56+
#if IS_BUILTIN(CONFIG_NET_ACT_MIRRED)
57+
if (a->ops->act == tcf_mirred_act)
58+
return tcf_mirred_act(skb, a, res);
59+
#endif
60+
#if IS_BUILTIN(CONFIG_NET_ACT_PEDIT)
61+
if (a->ops->act == tcf_pedit_act)
62+
return tcf_pedit_act(skb, a, res);
63+
#endif
64+
#if IS_BUILTIN(CONFIG_NET_ACT_SKBEDIT)
65+
if (a->ops->act == tcf_skbedit_act)
66+
return tcf_skbedit_act(skb, a, res);
67+
#endif
68+
#if IS_BUILTIN(CONFIG_NET_ACT_SKBMOD)
69+
if (a->ops->act == tcf_skbmod_act)
70+
return tcf_skbmod_act(skb, a, res);
71+
#endif
72+
#if IS_BUILTIN(CONFIG_NET_ACT_POLICE)
73+
if (a->ops->act == tcf_police_act)
74+
return tcf_police_act(skb, a, res);
75+
#endif
76+
#if IS_BUILTIN(CONFIG_NET_ACT_BPF)
77+
if (a->ops->act == tcf_bpf_act)
78+
return tcf_bpf_act(skb, a, res);
79+
#endif
80+
#if IS_BUILTIN(CONFIG_NET_ACT_CONNMARK)
81+
if (a->ops->act == tcf_connmark_act)
82+
return tcf_connmark_act(skb, a, res);
83+
#endif
84+
#if IS_BUILTIN(CONFIG_NET_ACT_CSUM)
85+
if (a->ops->act == tcf_csum_act)
86+
return tcf_csum_act(skb, a, res);
87+
#endif
88+
#if IS_BUILTIN(CONFIG_NET_ACT_CT)
89+
if (a->ops->act == tcf_ct_act)
90+
return tcf_ct_act(skb, a, res);
91+
#endif
92+
#if IS_BUILTIN(CONFIG_NET_ACT_CTINFO)
93+
if (a->ops->act == tcf_ctinfo_act)
94+
return tcf_ctinfo_act(skb, a, res);
95+
#endif
96+
#if IS_BUILTIN(CONFIG_NET_ACT_GATE)
97+
if (a->ops->act == tcf_gate_act)
98+
return tcf_gate_act(skb, a, res);
99+
#endif
100+
#if IS_BUILTIN(CONFIG_NET_ACT_MPLS)
101+
if (a->ops->act == tcf_mpls_act)
102+
return tcf_mpls_act(skb, a, res);
103+
#endif
104+
#if IS_BUILTIN(CONFIG_NET_ACT_NAT)
105+
if (a->ops->act == tcf_nat_act)
106+
return tcf_nat_act(skb, a, res);
107+
#endif
108+
#if IS_BUILTIN(CONFIG_NET_ACT_TUNNEL_KEY)
109+
if (a->ops->act == tunnel_key_act)
110+
return tunnel_key_act(skb, a, res);
111+
#endif
112+
#if IS_BUILTIN(CONFIG_NET_ACT_VLAN)
113+
if (a->ops->act == tcf_vlan_act)
114+
return tcf_vlan_act(skb, a, res);
115+
#endif
116+
#if IS_BUILTIN(CONFIG_NET_ACT_IFE)
117+
if (a->ops->act == tcf_ife_act)
118+
return tcf_ife_act(skb, a, res);
119+
#endif
120+
#if IS_BUILTIN(CONFIG_NET_ACT_IPT)
121+
if (a->ops->act == tcf_ipt_act)
122+
return tcf_ipt_act(skb, a, res);
123+
#endif
124+
#if IS_BUILTIN(CONFIG_NET_ACT_SIMP)
125+
if (a->ops->act == tcf_simp_act)
126+
return tcf_simp_act(skb, a, res);
127+
#endif
128+
#if IS_BUILTIN(CONFIG_NET_ACT_SAMPLE)
129+
if (a->ops->act == tcf_sample_act)
130+
return tcf_sample_act(skb, a, res);
131+
#endif
132+
133+
skip:
134+
return a->ops->act(skb, a, res);
135+
}
136+
137+
#endif /* CONFIG_NET_CLS_ACT */
138+
139+
/* TC Filters */
140+
#ifdef CONFIG_NET_CLS
141+
142+
#define TC_INDIRECT_FILTER_DECLARE(fname) \
143+
INDIRECT_CALLABLE_DECLARE(int fname(struct sk_buff *skb, \
144+
const struct tcf_proto *tp, \
145+
struct tcf_result *res))
146+
147+
TC_INDIRECT_FILTER_DECLARE(basic_classify);
148+
TC_INDIRECT_FILTER_DECLARE(cls_bpf_classify);
149+
TC_INDIRECT_FILTER_DECLARE(cls_cgroup_classify);
150+
TC_INDIRECT_FILTER_DECLARE(fl_classify);
151+
TC_INDIRECT_FILTER_DECLARE(flow_classify);
152+
TC_INDIRECT_FILTER_DECLARE(fw_classify);
153+
TC_INDIRECT_FILTER_DECLARE(mall_classify);
154+
TC_INDIRECT_FILTER_DECLARE(route4_classify);
155+
TC_INDIRECT_FILTER_DECLARE(rsvp_classify);
156+
TC_INDIRECT_FILTER_DECLARE(rsvp6_classify);
157+
TC_INDIRECT_FILTER_DECLARE(tcindex_classify);
158+
TC_INDIRECT_FILTER_DECLARE(u32_classify);
159+
160+
static inline int tc_classify(struct sk_buff *skb, const struct tcf_proto *tp,
161+
struct tcf_result *res)
162+
{
163+
if (static_branch_likely(&tc_skip_wrapper))
164+
goto skip;
165+
166+
#if IS_BUILTIN(CONFIG_NET_CLS_BPF)
167+
if (tp->classify == cls_bpf_classify)
168+
return cls_bpf_classify(skb, tp, res);
169+
#endif
170+
#if IS_BUILTIN(CONFIG_NET_CLS_U32)
171+
if (tp->classify == u32_classify)
172+
return u32_classify(skb, tp, res);
173+
#endif
174+
#if IS_BUILTIN(CONFIG_NET_CLS_FLOWER)
175+
if (tp->classify == fl_classify)
176+
return fl_classify(skb, tp, res);
177+
#endif
178+
#if IS_BUILTIN(CONFIG_NET_CLS_FW)
179+
if (tp->classify == fw_classify)
180+
return fw_classify(skb, tp, res);
181+
#endif
182+
#if IS_BUILTIN(CONFIG_NET_CLS_MATCHALL)
183+
if (tp->classify == mall_classify)
184+
return mall_classify(skb, tp, res);
185+
#endif
186+
#if IS_BUILTIN(CONFIG_NET_CLS_BASIC)
187+
if (tp->classify == basic_classify)
188+
return basic_classify(skb, tp, res);
189+
#endif
190+
#if IS_BUILTIN(CONFIG_NET_CLS_CGROUP)
191+
if (tp->classify == cls_cgroup_classify)
192+
return cls_cgroup_classify(skb, tp, res);
193+
#endif
194+
#if IS_BUILTIN(CONFIG_NET_CLS_FLOW)
195+
if (tp->classify == flow_classify)
196+
return flow_classify(skb, tp, res);
197+
#endif
198+
#if IS_BUILTIN(CONFIG_NET_CLS_ROUTE4)
199+
if (tp->classify == route4_classify)
200+
return route4_classify(skb, tp, res);
201+
#endif
202+
#if IS_BUILTIN(CONFIG_NET_CLS_RSVP)
203+
if (tp->classify == rsvp_classify)
204+
return rsvp_classify(skb, tp, res);
205+
#endif
206+
#if IS_BUILTIN(CONFIG_NET_CLS_RSVP6)
207+
if (tp->classify == rsvp6_classify)
208+
return rsvp6_classify(skb, tp, res);
209+
#endif
210+
#if IS_BUILTIN(CONFIG_NET_CLS_TCINDEX)
211+
if (tp->classify == tcindex_classify)
212+
return tcindex_classify(skb, tp, res);
213+
#endif
214+
215+
skip:
216+
return tp->classify(skb, tp, res);
217+
}
218+
219+
static inline void tc_wrapper_init(void)
220+
{
221+
#ifdef CONFIG_X86
222+
if (!cpu_feature_enabled(X86_FEATURE_RETPOLINE))
223+
static_branch_enable(&tc_skip_wrapper);
224+
#endif
225+
}
226+
227+
#endif /* CONFIG_NET_CLS */
228+
229+
#else
230+
231+
#define TC_INDIRECT_SCOPE static
232+
233+
static inline int tc_act(struct sk_buff *skb, const struct tc_action *a,
234+
struct tcf_result *res)
235+
{
236+
return a->ops->act(skb, a, res);
237+
}
238+
239+
static inline int tc_classify(struct sk_buff *skb, const struct tcf_proto *tp,
240+
struct tcf_result *res)
241+
{
242+
return tp->classify(skb, tp, res);
243+
}
244+
245+
static inline void tc_wrapper_init(void)
246+
{
247+
}
248+
249+
#endif
250+
251+
#endif /* __NET_TC_WRAPPER_H */

‎net/sched/sch_api.c

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
#include <net/netlink.h>
3232
#include <net/pkt_sched.h>
3333
#include <net/pkt_cls.h>
34+
#include <net/tc_wrapper.h>
3435

3536
#include <trace/events/qdisc.h>
3637

@@ -2273,6 +2274,8 @@ static struct pernet_operations psched_net_ops = {
22732274
.exit = psched_net_exit,
22742275
};
22752276

2277+
DEFINE_STATIC_KEY_FALSE(tc_skip_wrapper);
2278+
22762279
static int __init pktsched_init(void)
22772280
{
22782281
int err;
@@ -2300,6 +2303,8 @@ static int __init pktsched_init(void)
23002303
rtnl_register(PF_UNSPEC, RTM_GETTCLASS, tc_ctl_tclass, tc_dump_tclass,
23012304
0);
23022305

2306+
tc_wrapper_init();
2307+
23032308
return 0;
23042309
}
23052310

0 commit comments

Comments
 (0)