Skip to content

Commit da19267

Browse files
mrutland-armwilldeacon
mrutland-arm
authored and
willdeacon
committed
arm64: entry: mark entry code as noinstr
Functions in entry-common.c are marked as notrace and NOKPROBE_SYMBOL(), but they're still subject to other instrumentation which may rely on lockdep/rcu/context-tracking being up-to-date, and may cause nested exceptions (e.g. for WARN/BUG or KASAN's use of BRK) which will corrupt exceptions registers which have not yet been read. Prevent this by marking all functions in entry-common.c as noinstr to prevent compiler instrumentation. This also blacklists the functions for tracing and kprobes, so we don't need to handle that separately. Functions elsewhere will be dealt with in subsequent patches. Signed-off-by: Mark Rutland <[email protected]> Cc: Catalin Marinas <[email protected]> Cc: James Morse <[email protected]> Cc: Will Deacon <[email protected]> Link: https://lore.kernel.org/r/[email protected] Signed-off-by: Will Deacon <[email protected]>
1 parent 114e0a6 commit da19267

File tree

1 file changed

+25
-50
lines changed

1 file changed

+25
-50
lines changed

‎arch/arm64/kernel/entry-common.c

Lines changed: 25 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -17,40 +17,36 @@
1717
#include <asm/mmu.h>
1818
#include <asm/sysreg.h>
1919

20-
static void notrace el1_abort(struct pt_regs *regs, unsigned long esr)
20+
static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr)
2121
{
2222
unsigned long far = read_sysreg(far_el1);
2323

2424
local_daif_inherit(regs);
2525
far = untagged_addr(far);
2626
do_mem_abort(far, esr, regs);
2727
}
28-
NOKPROBE_SYMBOL(el1_abort);
2928

30-
static void notrace el1_pc(struct pt_regs *regs, unsigned long esr)
29+
static void noinstr el1_pc(struct pt_regs *regs, unsigned long esr)
3130
{
3231
unsigned long far = read_sysreg(far_el1);
3332

3433
local_daif_inherit(regs);
3534
do_sp_pc_abort(far, esr, regs);
3635
}
37-
NOKPROBE_SYMBOL(el1_pc);
3836

39-
static void notrace el1_undef(struct pt_regs *regs)
37+
static void noinstr el1_undef(struct pt_regs *regs)
4038
{
4139
local_daif_inherit(regs);
4240
do_undefinstr(regs);
4341
}
44-
NOKPROBE_SYMBOL(el1_undef);
4542

46-
static void notrace el1_inv(struct pt_regs *regs, unsigned long esr)
43+
static void noinstr el1_inv(struct pt_regs *regs, unsigned long esr)
4744
{
4845
local_daif_inherit(regs);
4946
bad_mode(regs, 0, esr);
5047
}
51-
NOKPROBE_SYMBOL(el1_inv);
5248

53-
static void notrace el1_dbg(struct pt_regs *regs, unsigned long esr)
49+
static void noinstr el1_dbg(struct pt_regs *regs, unsigned long esr)
5450
{
5551
unsigned long far = read_sysreg(far_el1);
5652

@@ -64,16 +60,14 @@ static void notrace el1_dbg(struct pt_regs *regs, unsigned long esr)
6460

6561
do_debug_exception(far, esr, regs);
6662
}
67-
NOKPROBE_SYMBOL(el1_dbg);
6863

69-
static void notrace el1_fpac(struct pt_regs *regs, unsigned long esr)
64+
static void noinstr el1_fpac(struct pt_regs *regs, unsigned long esr)
7065
{
7166
local_daif_inherit(regs);
7267
do_ptrauth_fault(regs, esr);
7368
}
74-
NOKPROBE_SYMBOL(el1_fpac);
7569

76-
asmlinkage void notrace el1_sync_handler(struct pt_regs *regs)
70+
asmlinkage void noinstr el1_sync_handler(struct pt_regs *regs)
7771
{
7872
unsigned long esr = read_sysreg(esr_el1);
7973

@@ -106,9 +100,8 @@ asmlinkage void notrace el1_sync_handler(struct pt_regs *regs)
106100
el1_inv(regs, esr);
107101
}
108102
}
109-
NOKPROBE_SYMBOL(el1_sync_handler);
110103

111-
static void notrace el0_da(struct pt_regs *regs, unsigned long esr)
104+
static void noinstr el0_da(struct pt_regs *regs, unsigned long esr)
112105
{
113106
unsigned long far = read_sysreg(far_el1);
114107

@@ -117,9 +110,8 @@ static void notrace el0_da(struct pt_regs *regs, unsigned long esr)
117110
far = untagged_addr(far);
118111
do_mem_abort(far, esr, regs);
119112
}
120-
NOKPROBE_SYMBOL(el0_da);
121113

122-
static void notrace el0_ia(struct pt_regs *regs, unsigned long esr)
114+
static void noinstr el0_ia(struct pt_regs *regs, unsigned long esr)
123115
{
124116
unsigned long far = read_sysreg(far_el1);
125117

@@ -135,41 +127,36 @@ static void notrace el0_ia(struct pt_regs *regs, unsigned long esr)
135127
local_daif_restore(DAIF_PROCCTX);
136128
do_mem_abort(far, esr, regs);
137129
}
138-
NOKPROBE_SYMBOL(el0_ia);
139130

140-
static void notrace el0_fpsimd_acc(struct pt_regs *regs, unsigned long esr)
131+
static void noinstr el0_fpsimd_acc(struct pt_regs *regs, unsigned long esr)
141132
{
142133
user_exit_irqoff();
143134
local_daif_restore(DAIF_PROCCTX);
144135
do_fpsimd_acc(esr, regs);
145136
}
146-
NOKPROBE_SYMBOL(el0_fpsimd_acc);
147137

148-
static void notrace el0_sve_acc(struct pt_regs *regs, unsigned long esr)
138+
static void noinstr el0_sve_acc(struct pt_regs *regs, unsigned long esr)
149139
{
150140
user_exit_irqoff();
151141
local_daif_restore(DAIF_PROCCTX);
152142
do_sve_acc(esr, regs);
153143
}
154-
NOKPROBE_SYMBOL(el0_sve_acc);
155144

156-
static void notrace el0_fpsimd_exc(struct pt_regs *regs, unsigned long esr)
145+
static void noinstr el0_fpsimd_exc(struct pt_regs *regs, unsigned long esr)
157146
{
158147
user_exit_irqoff();
159148
local_daif_restore(DAIF_PROCCTX);
160149
do_fpsimd_exc(esr, regs);
161150
}
162-
NOKPROBE_SYMBOL(el0_fpsimd_exc);
163151

164-
static void notrace el0_sys(struct pt_regs *regs, unsigned long esr)
152+
static void noinstr el0_sys(struct pt_regs *regs, unsigned long esr)
165153
{
166154
user_exit_irqoff();
167155
local_daif_restore(DAIF_PROCCTX);
168156
do_sysinstr(esr, regs);
169157
}
170-
NOKPROBE_SYMBOL(el0_sys);
171158

172-
static void notrace el0_pc(struct pt_regs *regs, unsigned long esr)
159+
static void noinstr el0_pc(struct pt_regs *regs, unsigned long esr)
173160
{
174161
unsigned long far = read_sysreg(far_el1);
175162

@@ -180,41 +167,36 @@ static void notrace el0_pc(struct pt_regs *regs, unsigned long esr)
180167
local_daif_restore(DAIF_PROCCTX);
181168
do_sp_pc_abort(far, esr, regs);
182169
}
183-
NOKPROBE_SYMBOL(el0_pc);
184170

185-
static void notrace el0_sp(struct pt_regs *regs, unsigned long esr)
171+
static void noinstr el0_sp(struct pt_regs *regs, unsigned long esr)
186172
{
187173
user_exit_irqoff();
188174
local_daif_restore(DAIF_PROCCTX);
189175
do_sp_pc_abort(regs->sp, esr, regs);
190176
}
191-
NOKPROBE_SYMBOL(el0_sp);
192177

193-
static void notrace el0_undef(struct pt_regs *regs)
178+
static void noinstr el0_undef(struct pt_regs *regs)
194179
{
195180
user_exit_irqoff();
196181
local_daif_restore(DAIF_PROCCTX);
197182
do_undefinstr(regs);
198183
}
199-
NOKPROBE_SYMBOL(el0_undef);
200184

201-
static void notrace el0_bti(struct pt_regs *regs)
185+
static void noinstr el0_bti(struct pt_regs *regs)
202186
{
203187
user_exit_irqoff();
204188
local_daif_restore(DAIF_PROCCTX);
205189
do_bti(regs);
206190
}
207-
NOKPROBE_SYMBOL(el0_bti);
208191

209-
static void notrace el0_inv(struct pt_regs *regs, unsigned long esr)
192+
static void noinstr el0_inv(struct pt_regs *regs, unsigned long esr)
210193
{
211194
user_exit_irqoff();
212195
local_daif_restore(DAIF_PROCCTX);
213196
bad_el0_sync(regs, 0, esr);
214197
}
215-
NOKPROBE_SYMBOL(el0_inv);
216198

217-
static void notrace el0_dbg(struct pt_regs *regs, unsigned long esr)
199+
static void noinstr el0_dbg(struct pt_regs *regs, unsigned long esr)
218200
{
219201
/* Only watchpoints write FAR_EL1, otherwise its UNKNOWN */
220202
unsigned long far = read_sysreg(far_el1);
@@ -226,26 +208,23 @@ static void notrace el0_dbg(struct pt_regs *regs, unsigned long esr)
226208
do_debug_exception(far, esr, regs);
227209
local_daif_restore(DAIF_PROCCTX_NOIRQ);
228210
}
229-
NOKPROBE_SYMBOL(el0_dbg);
230211

231-
static void notrace el0_svc(struct pt_regs *regs)
212+
static void noinstr el0_svc(struct pt_regs *regs)
232213
{
233214
if (system_uses_irq_prio_masking())
234215
gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);
235216

236217
do_el0_svc(regs);
237218
}
238-
NOKPROBE_SYMBOL(el0_svc);
239219

240-
static void notrace el0_fpac(struct pt_regs *regs, unsigned long esr)
220+
static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr)
241221
{
242222
user_exit_irqoff();
243223
local_daif_restore(DAIF_PROCCTX);
244224
do_ptrauth_fault(regs, esr);
245225
}
246-
NOKPROBE_SYMBOL(el0_fpac);
247226

248-
asmlinkage void notrace el0_sync_handler(struct pt_regs *regs)
227+
asmlinkage void noinstr el0_sync_handler(struct pt_regs *regs)
249228
{
250229
unsigned long esr = read_sysreg(esr_el1);
251230

@@ -297,27 +276,24 @@ asmlinkage void notrace el0_sync_handler(struct pt_regs *regs)
297276
el0_inv(regs, esr);
298277
}
299278
}
300-
NOKPROBE_SYMBOL(el0_sync_handler);
301279

302280
#ifdef CONFIG_COMPAT
303-
static void notrace el0_cp15(struct pt_regs *regs, unsigned long esr)
281+
static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr)
304282
{
305283
user_exit_irqoff();
306284
local_daif_restore(DAIF_PROCCTX);
307285
do_cp15instr(esr, regs);
308286
}
309-
NOKPROBE_SYMBOL(el0_cp15);
310287

311-
static void notrace el0_svc_compat(struct pt_regs *regs)
288+
static void noinstr el0_svc_compat(struct pt_regs *regs)
312289
{
313290
if (system_uses_irq_prio_masking())
314291
gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);
315292

316293
do_el0_svc_compat(regs);
317294
}
318-
NOKPROBE_SYMBOL(el0_svc_compat);
319295

320-
asmlinkage void notrace el0_sync_compat_handler(struct pt_regs *regs)
296+
asmlinkage void noinstr el0_sync_compat_handler(struct pt_regs *regs)
321297
{
322298
unsigned long esr = read_sysreg(esr_el1);
323299

@@ -360,5 +336,4 @@ asmlinkage void notrace el0_sync_compat_handler(struct pt_regs *regs)
360336
el0_inv(regs, esr);
361337
}
362338
}
363-
NOKPROBE_SYMBOL(el0_sync_compat_handler);
364339
#endif /* CONFIG_COMPAT */

0 commit comments

Comments
 (0)