Skip to content

Commit f19fbd5

Browse files
author
Martin Schwidefsky
committed
s390: introduce execute-trampolines for branches
Add CONFIG_EXPOLINE to enable the use of the new -mindirect-branch= and -mfunction_return= compiler options to create a kernel fortified against the specte v2 attack. With CONFIG_EXPOLINE=y all indirect branches will be issued with an execute type instruction. For z10 or newer the EXRL instruction will be used, for older machines the EX instruction. The typical indirect call basr %r14,%r1 is replaced with a PC relative call to a new thunk brasl %r14,__s390x_indirect_jump_r1 The thunk contains the EXRL/EX instruction to the indirect branch __s390x_indirect_jump_r1: exrl 0,0f j . 0: br %r1 The detour via the execute type instruction has a performance impact. To get rid of the detour the new kernel parameter "nospectre_v2" and "spectre_v2=[on,off,auto]" can be used. If the parameter is specified the kernel and module code will be patched at runtime. Signed-off-by: Martin Schwidefsky <[email protected]>
1 parent 6b73044 commit f19fbd5

File tree

12 files changed

+327
-35
lines changed

12 files changed

+327
-35
lines changed

‎arch/s390/Kconfig

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -557,6 +557,34 @@ config KERNEL_NOBP
557557

558558
If unsure, say N.
559559

560+
config EXPOLINE
561+
def_bool n
562+
prompt "Avoid speculative indirect branches in the kernel"
563+
help
564+
Compile the kernel with the expoline compiler options to guard
565+
against kernel-to-user data leaks by avoiding speculative indirect
566+
branches.
567+
Requires a compiler with -mindirect-branch=thunk support for full
568+
protection. The kernel may run slower.
569+
570+
If unsure, say N.
571+
572+
choice
573+
prompt "Expoline default"
574+
depends on EXPOLINE
575+
default EXPOLINE_FULL
576+
577+
config EXPOLINE_OFF
578+
bool "spectre_v2=off"
579+
580+
config EXPOLINE_MEDIUM
581+
bool "spectre_v2=auto"
582+
583+
config EXPOLINE_FULL
584+
bool "spectre_v2=on"
585+
586+
endchoice
587+
560588
endmenu
561589

562590
menu "Memory setup"

‎arch/s390/Makefile

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,16 @@ ifeq ($(call cc-option-yn,-mwarn-dynamicstack),y)
7878
cflags-$(CONFIG_WARN_DYNAMIC_STACK) += -mwarn-dynamicstack
7979
endif
8080

81+
ifdef CONFIG_EXPOLINE
82+
ifeq ($(call cc-option-yn,$(CC_FLAGS_MARCH) -mindirect-branch=thunk),y)
83+
CC_FLAGS_EXPOLINE := -mindirect-branch=thunk
84+
CC_FLAGS_EXPOLINE += -mfunction-return=thunk
85+
CC_FLAGS_EXPOLINE += -mindirect-branch-table
86+
export CC_FLAGS_EXPOLINE
87+
cflags-y += $(CC_FLAGS_EXPOLINE)
88+
endif
89+
endif
90+
8191
ifdef CONFIG_FUNCTION_TRACER
8292
# make use of hotpatch feature if the compiler supports it
8393
cc_hotpatch := -mhotpatch=0,3

‎arch/s390/include/asm/lowcore.h

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,11 @@ struct lowcore {
136136
__u64 vdso_per_cpu_data; /* 0x03b8 */
137137
__u64 machine_flags; /* 0x03c0 */
138138
__u64 gmap; /* 0x03c8 */
139-
__u8 pad_0x03d0[0x0e00-0x03d0]; /* 0x03d0 */
139+
__u8 pad_0x03d0[0x0400-0x03d0]; /* 0x03d0 */
140+
141+
/* br %r1 trampoline */
142+
__u16 br_r1_trampoline; /* 0x0400 */
143+
__u8 pad_0x0402[0x0e00-0x0402]; /* 0x0402 */
140144

141145
/*
142146
* 0xe00 contains the address of the IPL Parameter Information

‎arch/s390/include/asm/nospec-branch.h

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
/* SPDX-License-Identifier: GPL-2.0 */
2+
#ifndef _ASM_S390_EXPOLINE_H
3+
#define _ASM_S390_EXPOLINE_H
4+
5+
#ifndef __ASSEMBLY__
6+
7+
#include <linux/types.h>
8+
9+
extern int nospec_call_disable;
10+
extern int nospec_return_disable;
11+
12+
void nospec_init_branches(void);
13+
void nospec_call_revert(s32 *start, s32 *end);
14+
void nospec_return_revert(s32 *start, s32 *end);
15+
16+
#endif /* __ASSEMBLY__ */
17+
18+
#endif /* _ASM_S390_EXPOLINE_H */

‎arch/s390/kernel/Makefile

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ UBSAN_SANITIZE_early.o := n
2929
#
3030
ifneq ($(CC_FLAGS_MARCH),-march=z900)
3131
CFLAGS_REMOVE_als.o += $(CC_FLAGS_MARCH)
32+
CFLAGS_REMOVE_als.o += $(CC_FLAGS_EXPOLINE)
3233
CFLAGS_als.o += -march=z900
3334
AFLAGS_REMOVE_head.o += $(CC_FLAGS_MARCH)
3435
AFLAGS_head.o += -march=z900
@@ -63,6 +64,9 @@ obj-y += entry.o reipl.o relocate_kernel.o kdebugfs.o alternative.o
6364

6465
extra-y += head.o head64.o vmlinux.lds
6566

67+
obj-$(CONFIG_EXPOLINE) += nospec-branch.o
68+
CFLAGS_REMOVE_expoline.o += $(CC_FLAGS_EXPOLINE)
69+
6670
obj-$(CONFIG_MODULES) += module.o
6771
obj-$(CONFIG_SMP) += smp.o
6872
obj-$(CONFIG_SCHED_TOPOLOGY) += topology.o

‎arch/s390/kernel/entry.S

Lines changed: 88 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -222,6 +222,68 @@ _PIF_WORK = (_PIF_PER_TRAP | _PIF_SYSCALL_RESTART)
222222
.popsection
223223
.endm
224224

225+
#ifdef CONFIG_EXPOLINE
226+
227+
.macro GEN_BR_THUNK name,reg,tmp
228+
.section .text.\name,"axG",@progbits,\name,comdat
229+
.globl \name
230+
.hidden \name
231+
.type \name,@function
232+
\name:
233+
.cfi_startproc
234+
#ifdef CONFIG_HAVE_MARCH_Z10_FEATURES
235+
exrl 0,0f
236+
#else
237+
larl \tmp,0f
238+
ex 0,0(\tmp)
239+
#endif
240+
j .
241+
0: br \reg
242+
.cfi_endproc
243+
.endm
244+
245+
GEN_BR_THUNK __s390x_indirect_jump_r1use_r9,%r9,%r1
246+
GEN_BR_THUNK __s390x_indirect_jump_r1use_r14,%r14,%r1
247+
GEN_BR_THUNK __s390x_indirect_jump_r11use_r14,%r14,%r11
248+
249+
.macro BASR_R14_R9
250+
0: brasl %r14,__s390x_indirect_jump_r1use_r9
251+
.pushsection .s390_indirect_branches,"a",@progbits
252+
.long 0b-.
253+
.popsection
254+
.endm
255+
256+
.macro BR_R1USE_R14
257+
0: jg __s390x_indirect_jump_r1use_r14
258+
.pushsection .s390_indirect_branches,"a",@progbits
259+
.long 0b-.
260+
.popsection
261+
.endm
262+
263+
.macro BR_R11USE_R14
264+
0: jg __s390x_indirect_jump_r11use_r14
265+
.pushsection .s390_indirect_branches,"a",@progbits
266+
.long 0b-.
267+
.popsection
268+
.endm
269+
270+
#else /* CONFIG_EXPOLINE */
271+
272+
.macro BASR_R14_R9
273+
basr %r14,%r9
274+
.endm
275+
276+
.macro BR_R1USE_R14
277+
br %r14
278+
.endm
279+
280+
.macro BR_R11USE_R14
281+
br %r14
282+
.endm
283+
284+
#endif /* CONFIG_EXPOLINE */
285+
286+
225287
.section .kprobes.text, "ax"
226288
.Ldummy:
227289
/*
@@ -237,7 +299,7 @@ _PIF_WORK = (_PIF_PER_TRAP | _PIF_SYSCALL_RESTART)
237299
ENTRY(__bpon)
238300
.globl __bpon
239301
BPON
240-
br %r14
302+
BR_R1USE_R14
241303

242304
/*
243305
* Scheduler resume function, called by switch_to
@@ -261,9 +323,9 @@ ENTRY(__switch_to)
261323
mvc __LC_CURRENT_PID(4,%r0),0(%r3) # store pid of next
262324
lmg %r6,%r15,__SF_GPRS(%r15) # load gprs of next task
263325
TSTMSK __LC_MACHINE_FLAGS,MACHINE_FLAG_LPP
264-
bzr %r14
326+
jz 0f
265327
.insn s,0xb2800000,__LC_LPP # set program parameter
266-
br %r14
328+
0: BR_R1USE_R14
267329

268330
.L__critical_start:
269331

@@ -330,7 +392,7 @@ sie_exit:
330392
xgr %r5,%r5
331393
lmg %r6,%r14,__SF_GPRS(%r15) # restore kernel registers
332394
lg %r2,__SF_EMPTY+16(%r15) # return exit reason code
333-
br %r14
395+
BR_R1USE_R14
334396
.Lsie_fault:
335397
lghi %r14,-EFAULT
336398
stg %r14,__SF_EMPTY+16(%r15) # set exit reason code
@@ -389,7 +451,7 @@ ENTRY(system_call)
389451
lgf %r9,0(%r8,%r10) # get system call add.
390452
TSTMSK __TI_flags(%r12),_TIF_TRACE
391453
jnz .Lsysc_tracesys
392-
basr %r14,%r9 # call sys_xxxx
454+
BASR_R14_R9 # call sys_xxxx
393455
stg %r2,__PT_R2(%r11) # store return value
394456

395457
.Lsysc_return:
@@ -574,7 +636,7 @@ ENTRY(system_call)
574636
lmg %r3,%r7,__PT_R3(%r11)
575637
stg %r7,STACK_FRAME_OVERHEAD(%r15)
576638
lg %r2,__PT_ORIG_GPR2(%r11)
577-
basr %r14,%r9 # call sys_xxx
639+
BASR_R14_R9 # call sys_xxx
578640
stg %r2,__PT_R2(%r11) # store return value
579641
.Lsysc_tracenogo:
580642
TSTMSK __TI_flags(%r12),_TIF_TRACE
@@ -598,7 +660,7 @@ ENTRY(ret_from_fork)
598660
lmg %r9,%r10,__PT_R9(%r11) # load gprs
599661
ENTRY(kernel_thread_starter)
600662
la %r2,0(%r10)
601-
basr %r14,%r9
663+
BASR_R14_R9
602664
j .Lsysc_tracenogo
603665

604666
/*
@@ -678,9 +740,9 @@ ENTRY(pgm_check_handler)
678740
nill %r10,0x007f
679741
sll %r10,2
680742
je .Lpgm_return
681-
lgf %r1,0(%r10,%r1) # load address of handler routine
743+
lgf %r9,0(%r10,%r1) # load address of handler routine
682744
lgr %r2,%r11 # pass pointer to pt_regs
683-
basr %r14,%r1 # branch to interrupt-handler
745+
BASR_R14_R9 # branch to interrupt-handler
684746
.Lpgm_return:
685747
LOCKDEP_SYS_EXIT
686748
tm __PT_PSW+1(%r11),0x01 # returning to user ?
@@ -998,7 +1060,7 @@ ENTRY(psw_idle)
9981060
stpt __TIMER_IDLE_ENTER(%r2)
9991061
.Lpsw_idle_lpsw:
10001062
lpswe __SF_EMPTY(%r15)
1001-
br %r14
1063+
BR_R1USE_R14
10021064
.Lpsw_idle_end:
10031065

10041066
/*
@@ -1012,7 +1074,7 @@ ENTRY(save_fpu_regs)
10121074
lg %r2,__LC_CURRENT
10131075
aghi %r2,__TASK_thread
10141076
TSTMSK __LC_CPU_FLAGS,_CIF_FPU
1015-
bor %r14
1077+
jo .Lsave_fpu_regs_exit
10161078
stfpc __THREAD_FPU_fpc(%r2)
10171079
lg %r3,__THREAD_FPU_regs(%r2)
10181080
TSTMSK __LC_MACHINE_FLAGS,MACHINE_FLAG_VX
@@ -1039,7 +1101,8 @@ ENTRY(save_fpu_regs)
10391101
std 15,120(%r3)
10401102
.Lsave_fpu_regs_done:
10411103
oi __LC_CPU_FLAGS+7,_CIF_FPU
1042-
br %r14
1104+
.Lsave_fpu_regs_exit:
1105+
BR_R1USE_R14
10431106
.Lsave_fpu_regs_end:
10441107
EXPORT_SYMBOL(save_fpu_regs)
10451108

@@ -1057,7 +1120,7 @@ load_fpu_regs:
10571120
lg %r4,__LC_CURRENT
10581121
aghi %r4,__TASK_thread
10591122
TSTMSK __LC_CPU_FLAGS,_CIF_FPU
1060-
bnor %r14
1123+
jno .Lload_fpu_regs_exit
10611124
lfpc __THREAD_FPU_fpc(%r4)
10621125
TSTMSK __LC_MACHINE_FLAGS,MACHINE_FLAG_VX
10631126
lg %r4,__THREAD_FPU_regs(%r4) # %r4 <- reg save area
@@ -1084,7 +1147,8 @@ load_fpu_regs:
10841147
ld 15,120(%r4)
10851148
.Lload_fpu_regs_done:
10861149
ni __LC_CPU_FLAGS+7,255-_CIF_FPU
1087-
br %r14
1150+
.Lload_fpu_regs_exit:
1151+
BR_R1USE_R14
10881152
.Lload_fpu_regs_end:
10891153

10901154
.L__critical_end:
@@ -1301,7 +1365,7 @@ cleanup_critical:
13011365
jl 0f
13021366
clg %r9,BASED(.Lcleanup_table+104) # .Lload_fpu_regs_end
13031367
jl .Lcleanup_load_fpu_regs
1304-
0: br %r14
1368+
0: BR_R11USE_R14
13051369

13061370
.align 8
13071371
.Lcleanup_table:
@@ -1337,7 +1401,7 @@ cleanup_critical:
13371401
ni __SIE_PROG0C+3(%r9),0xfe # no longer in SIE
13381402
lctlg %c1,%c1,__LC_USER_ASCE # load primary asce
13391403
larl %r9,sie_exit # skip forward to sie_exit
1340-
br %r14
1404+
BR_R11USE_R14
13411405
#endif
13421406

13431407
.Lcleanup_system_call:
@@ -1390,7 +1454,7 @@ cleanup_critical:
13901454
stg %r15,56(%r11) # r15 stack pointer
13911455
# set new psw address and exit
13921456
larl %r9,.Lsysc_do_svc
1393-
br %r14
1457+
BR_R11USE_R14
13941458
.Lcleanup_system_call_insn:
13951459
.quad system_call
13961460
.quad .Lsysc_stmg
@@ -1402,7 +1466,7 @@ cleanup_critical:
14021466

14031467
.Lcleanup_sysc_tif:
14041468
larl %r9,.Lsysc_tif
1405-
br %r14
1469+
BR_R11USE_R14
14061470

14071471
.Lcleanup_sysc_restore:
14081472
# check if stpt has been executed
@@ -1419,14 +1483,14 @@ cleanup_critical:
14191483
mvc 0(64,%r11),__PT_R8(%r9)
14201484
lmg %r0,%r7,__PT_R0(%r9)
14211485
1: lmg %r8,%r9,__LC_RETURN_PSW
1422-
br %r14
1486+
BR_R11USE_R14
14231487
.Lcleanup_sysc_restore_insn:
14241488
.quad .Lsysc_exit_timer
14251489
.quad .Lsysc_done - 4
14261490

14271491
.Lcleanup_io_tif:
14281492
larl %r9,.Lio_tif
1429-
br %r14
1493+
BR_R11USE_R14
14301494

14311495
.Lcleanup_io_restore:
14321496
# check if stpt has been executed
@@ -1440,7 +1504,7 @@ cleanup_critical:
14401504
mvc 0(64,%r11),__PT_R8(%r9)
14411505
lmg %r0,%r7,__PT_R0(%r9)
14421506
1: lmg %r8,%r9,__LC_RETURN_PSW
1443-
br %r14
1507+
BR_R11USE_R14
14441508
.Lcleanup_io_restore_insn:
14451509
.quad .Lio_exit_timer
14461510
.quad .Lio_done - 4
@@ -1493,17 +1557,17 @@ cleanup_critical:
14931557
# prepare return psw
14941558
nihh %r8,0xfcfd # clear irq & wait state bits
14951559
lg %r9,48(%r11) # return from psw_idle
1496-
br %r14
1560+
BR_R11USE_R14
14971561
.Lcleanup_idle_insn:
14981562
.quad .Lpsw_idle_lpsw
14991563

15001564
.Lcleanup_save_fpu_regs:
15011565
larl %r9,save_fpu_regs
1502-
br %r14
1566+
BR_R11USE_R14
15031567

15041568
.Lcleanup_load_fpu_regs:
15051569
larl %r9,load_fpu_regs
1506-
br %r14
1570+
BR_R11USE_R14
15071571

15081572
/*
15091573
* Integer constants
@@ -1523,7 +1587,6 @@ cleanup_critical:
15231587
.Lsie_crit_mcck_length:
15241588
.quad .Lsie_skip - .Lsie_entry
15251589
#endif
1526-
15271590
.section .rodata, "a"
15281591
#define SYSCALL(esame,emu) .long esame
15291592
.globl sys_call_table

0 commit comments

Comments
 (0)