Skip to content

Commit 105fc33

Browse files
mrutland-armwilldeacon
authored andcommitted
arm64: entry: move el1 irq/nmi logic to C
In preparation for reworking the EL1 irq/nmi entry code, move the existing logic to C. We no longer need the asm_nmi_enter() and asm_nmi_exit() wrappers, so these are removed. The new C functions are marked noinstr, which prevents compiler instrumentation and runtime probing. In subsequent patches we'll want the new C helpers to be called in all cases, so we don't bother wrapping the calls with ifdeferry. Even when the new C functions are stubs the trivial calls are unlikely to have a measurable impact on the IRQ or NMI paths anyway. Prototypes are added to <asm/exception.h> as otherwise (in some configurations) GCC will complain about the lack of a forward declaration. We already do this for existing function, e.g. enter_from_user_mode(). The new helpers are marked as noinstr (which prevents all instrumentation, tracing, and kprobes). Otherwise, there should be no functional change as a result of this patch. Signed-off-by: Mark Rutland <[email protected]> Cc: Catalin Marinas <[email protected]> Cc: James Morse <[email protected]> Cc: Will Deacon <[email protected]> Link: https://lore.kernel.org/r/[email protected] Signed-off-by: Will Deacon <[email protected]>
1 parent 3cb5ed4 commit 105fc33

File tree

4 files changed

+22
-45
lines changed

4 files changed

+22
-45
lines changed

arch/arm64/include/asm/exception.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@ static inline u32 disr_to_esr(u64 disr)
3131
return esr;
3232
}
3333

34+
asmlinkage void noinstr enter_el1_irq_or_nmi(struct pt_regs *regs);
35+
asmlinkage void noinstr exit_el1_irq_or_nmi(struct pt_regs *regs);
3436
asmlinkage void enter_from_user_mode(void);
3537
void do_mem_abort(unsigned long addr, unsigned int esr, struct pt_regs *regs);
3638
void do_undefinstr(struct pt_regs *regs);

arch/arm64/kernel/entry-common.c

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,22 @@
1717
#include <asm/mmu.h>
1818
#include <asm/sysreg.h>
1919

20+
asmlinkage void noinstr enter_el1_irq_or_nmi(struct pt_regs *regs)
21+
{
22+
if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs))
23+
nmi_enter();
24+
25+
trace_hardirqs_off();
26+
}
27+
28+
asmlinkage void noinstr exit_el1_irq_or_nmi(struct pt_regs *regs)
29+
{
30+
if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs))
31+
nmi_exit();
32+
else
33+
trace_hardirqs_on();
34+
}
35+
2036
static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr)
2137
{
2238
unsigned long far = read_sysreg(far_el1);

arch/arm64/kernel/entry.S

Lines changed: 4 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -637,16 +637,8 @@ SYM_CODE_START_LOCAL_NOALIGN(el1_irq)
637637
gic_prio_irq_setup pmr=x20, tmp=x1
638638
enable_da_f
639639

640-
#ifdef CONFIG_ARM64_PSEUDO_NMI
641-
test_irqs_unmasked res=x0, pmr=x20
642-
cbz x0, 1f
643-
bl asm_nmi_enter
644-
1:
645-
#endif
646-
647-
#ifdef CONFIG_TRACE_IRQFLAGS
648-
bl trace_hardirqs_off
649-
#endif
640+
mov x0, sp
641+
bl enter_el1_irq_or_nmi
650642

651643
irq_handler
652644

@@ -665,26 +657,8 @@ alternative_else_nop_endif
665657
1:
666658
#endif
667659

668-
#ifdef CONFIG_ARM64_PSEUDO_NMI
669-
/*
670-
* When using IRQ priority masking, we can get spurious interrupts while
671-
* PMR is set to GIC_PRIO_IRQOFF. An NMI might also have occurred in a
672-
* section with interrupts disabled. Skip tracing in those cases.
673-
*/
674-
test_irqs_unmasked res=x0, pmr=x20
675-
cbz x0, 1f
676-
bl asm_nmi_exit
677-
1:
678-
#endif
679-
680-
#ifdef CONFIG_TRACE_IRQFLAGS
681-
#ifdef CONFIG_ARM64_PSEUDO_NMI
682-
test_irqs_unmasked res=x0, pmr=x20
683-
cbnz x0, 1f
684-
#endif
685-
bl trace_hardirqs_on
686-
1:
687-
#endif
660+
mov x0, sp
661+
bl exit_el1_irq_or_nmi
688662

689663
kernel_exit 1
690664
SYM_CODE_END(el1_irq)

arch/arm64/kernel/irq.c

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -67,18 +67,3 @@ void __init init_IRQ(void)
6767
local_daif_restore(DAIF_PROCCTX_NOIRQ);
6868
}
6969
}
70-
71-
/*
72-
* Stubs to make nmi_enter/exit() code callable from ASM
73-
*/
74-
asmlinkage void notrace asm_nmi_enter(void)
75-
{
76-
nmi_enter();
77-
}
78-
NOKPROBE_SYMBOL(asm_nmi_enter);
79-
80-
asmlinkage void notrace asm_nmi_exit(void)
81-
{
82-
nmi_exit();
83-
}
84-
NOKPROBE_SYMBOL(asm_nmi_exit);

0 commit comments

Comments
 (0)