Skip to content

Commit dc7ee00

Browse files
author
Martin Schwidefsky
committed
s390: lowcore stack pointer offsets
Store the stack pointers in the lowcore for the kernel stack, the async stack and the panic stack with the offset required for the first user. This avoids an unnecessary add instruction on the system call path. Signed-off-by: Martin Schwidefsky <[email protected]>
1 parent b8668fd commit dc7ee00

File tree

4 files changed

+46
-48
lines changed

4 files changed

+46
-48
lines changed

‎arch/s390/kernel/entry.S

Lines changed: 16 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ _TIF_TRACE = (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \
4545

4646
STACK_SHIFT = PAGE_SHIFT + THREAD_ORDER
4747
STACK_SIZE = 1 << STACK_SHIFT
48+
STACK_INIT = STACK_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE
4849

4950
#define BASED(name) name-system_call(%r13)
5051

@@ -97,10 +98,10 @@ STACK_SIZE = 1 << STACK_SHIFT
9798
sra %r14,\shift
9899
jnz 1f
99100
CHECK_STACK 1<<\shift,\savearea
101+
ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
100102
j 2f
101103
1: l %r15,\stack # load target stack
102-
2: ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
103-
la %r11,STACK_FRAME_OVERHEAD(%r15)
104+
2: la %r11,STACK_FRAME_OVERHEAD(%r15)
104105
.endm
105106

106107
.macro ADD64 high,low,timer
@@ -150,7 +151,7 @@ ENTRY(__switch_to)
150151
l %r4,__THREAD_info(%r2) # get thread_info of prev
151152
l %r5,__THREAD_info(%r3) # get thread_info of next
152153
lr %r15,%r5
153-
ahi %r15,STACK_SIZE # end of kernel stack of next
154+
ahi %r15,STACK_INIT # end of kernel stack of next
154155
st %r3,__LC_CURRENT # store task struct of next
155156
st %r5,__LC_THREAD_INFO # store thread info of next
156157
st %r15,__LC_KERNEL_STACK # store end of kernel stack
@@ -178,7 +179,6 @@ sysc_stm:
178179
l %r13,__LC_SVC_NEW_PSW+4
179180
sysc_per:
180181
l %r15,__LC_KERNEL_STACK
181-
ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
182182
la %r11,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs
183183
sysc_vtime:
184184
UPDATE_VTIME %r8,%r9,__LC_SYNC_ENTER_TIMER
@@ -359,11 +359,11 @@ ENTRY(pgm_check_handler)
359359
tm __LC_PGM_ILC+3,0x80 # check for per exception
360360
jnz pgm_svcper # -> single stepped svc
361361
0: CHECK_STACK STACK_SIZE,__LC_SAVE_AREA_SYNC
362+
ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
362363
j 2f
363364
1: UPDATE_VTIME %r14,%r15,__LC_SYNC_ENTER_TIMER
364365
l %r15,__LC_KERNEL_STACK
365-
2: ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
366-
la %r11,STACK_FRAME_OVERHEAD(%r15)
366+
2: la %r11,STACK_FRAME_OVERHEAD(%r15)
367367
stm %r0,%r7,__PT_R0(%r11)
368368
mvc __PT_R8(32,%r11),__LC_SAVE_AREA_SYNC
369369
stm %r8,%r9,__PT_PSW(%r11)
@@ -485,7 +485,6 @@ io_work:
485485
#
486486
io_work_user:
487487
l %r1,__LC_KERNEL_STACK
488-
ahi %r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
489488
mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
490489
xc __SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1)
491490
la %r11,STACK_FRAME_OVERHEAD(%r1)
@@ -646,7 +645,6 @@ mcck_skip:
646645
tm __PT_PSW+1(%r11),0x01 # returning to user ?
647646
jno mcck_return
648647
l %r1,__LC_KERNEL_STACK # switch to kernel stack
649-
ahi %r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
650648
mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
651649
xc __SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1)
652650
la %r11,STACK_FRAME_OVERHEAD(%r15)
@@ -674,6 +672,7 @@ mcck_panic:
674672
sra %r14,PAGE_SHIFT
675673
jz 0f
676674
l %r15,__LC_PANIC_STACK
675+
j mcck_skip
677676
0: ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
678677
j mcck_skip
679678

@@ -714,12 +713,10 @@ ENTRY(restart_int_handler)
714713
*/
715714
stack_overflow:
716715
l %r15,__LC_PANIC_STACK # change to panic stack
717-
ahi %r15,-__PT_SIZE # create pt_regs
718-
stm %r0,%r7,__PT_R0(%r15)
719-
stm %r8,%r9,__PT_PSW(%r15)
716+
la %r11,STACK_FRAME_OVERHEAD(%r15)
717+
stm %r0,%r7,__PT_R0(%r11)
718+
stm %r8,%r9,__PT_PSW(%r11)
720719
mvc __PT_R8(32,%r11),0(%r14)
721-
lr %r15,%r11
722-
ahi %r15,-STACK_FRAME_OVERHEAD
723720
l %r1,BASED(1f)
724721
xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
725722
lr %r2,%r11 # pass pointer to pt_regs
@@ -799,15 +796,14 @@ cleanup_system_call:
799796
mvc __LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER
800797
# set up saved register 11
801798
l %r15,__LC_KERNEL_STACK
802-
ahi %r15,-__PT_SIZE
803-
st %r15,12(%r11) # r11 pt_regs pointer
799+
la %r9,STACK_FRAME_OVERHEAD(%r15)
800+
st %r9,12(%r11) # r11 pt_regs pointer
804801
# fill pt_regs
805-
mvc __PT_R8(32,%r15),__LC_SAVE_AREA_SYNC
806-
stm %r0,%r7,__PT_R0(%r15)
807-
mvc __PT_PSW(8,%r15),__LC_SVC_OLD_PSW
808-
mvc __PT_INT_CODE(4,%r15),__LC_SVC_ILC
802+
mvc __PT_R8(32,%r9),__LC_SAVE_AREA_SYNC
803+
stm %r0,%r7,__PT_R0(%r9)
804+
mvc __PT_PSW(8,%r9),__LC_SVC_OLD_PSW
805+
mvc __PT_INT_CODE(4,%r9),__LC_SVC_ILC
809806
# setup saved register 15
810-
ahi %r15,-STACK_FRAME_OVERHEAD
811807
st %r15,28(%r11) # r15 stack pointer
812808
# set new psw address and exit
813809
l %r9,BASED(cleanup_table+4) # sysc_do_svc + 0x80000000

‎arch/s390/kernel/entry64.S

Lines changed: 14 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ __PT_R15 = __PT_GPRS + 120
3939

4040
STACK_SHIFT = PAGE_SHIFT + THREAD_ORDER
4141
STACK_SIZE = 1 << STACK_SHIFT
42+
STACK_INIT = STACK_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE
4243

4344
_TIF_WORK_SVC = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \
4445
_TIF_MCCK_PENDING | _TIF_PER_TRAP )
@@ -124,10 +125,10 @@ _TIF_EXIT_SIE = (_TIF_SIGPENDING | _TIF_NEED_RESCHED | _TIF_MCCK_PENDING)
124125
srag %r14,%r14,\shift
125126
jnz 1f
126127
CHECK_STACK 1<<\shift,\savearea
128+
aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
127129
j 2f
128130
1: lg %r15,\stack # load target stack
129-
2: aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
130-
la %r11,STACK_FRAME_OVERHEAD(%r15)
131+
2: la %r11,STACK_FRAME_OVERHEAD(%r15)
131132
.endm
132133

133134
.macro UPDATE_VTIME scratch,enter_timer
@@ -177,7 +178,7 @@ ENTRY(__switch_to)
177178
lg %r4,__THREAD_info(%r2) # get thread_info of prev
178179
lg %r5,__THREAD_info(%r3) # get thread_info of next
179180
lgr %r15,%r5
180-
aghi %r15,STACK_SIZE # end of kernel stack of next
181+
aghi %r15,STACK_INIT # end of kernel stack of next
181182
stg %r3,__LC_CURRENT # store task struct of next
182183
stg %r5,__LC_THREAD_INFO # store thread info of next
183184
stg %r15,__LC_KERNEL_STACK # store end of kernel stack
@@ -203,10 +204,8 @@ sysc_stmg:
203204
stmg %r8,%r15,__LC_SAVE_AREA_SYNC
204205
lg %r10,__LC_LAST_BREAK
205206
lg %r12,__LC_THREAD_INFO
206-
larl %r13,system_call
207207
sysc_per:
208208
lg %r15,__LC_KERNEL_STACK
209-
aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
210209
la %r11,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs
211210
sysc_vtime:
212211
UPDATE_VTIME %r13,__LC_SYNC_ENTER_TIMER
@@ -389,6 +388,7 @@ ENTRY(pgm_check_handler)
389388
tm __LC_PGM_ILC+3,0x80 # check for per exception
390389
jnz pgm_svcper # -> single stepped svc
391390
0: CHECK_STACK STACK_SIZE,__LC_SAVE_AREA_SYNC
391+
aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
392392
j 2f
393393
1: UPDATE_VTIME %r14,__LC_SYNC_ENTER_TIMER
394394
LAST_BREAK %r14
@@ -398,8 +398,7 @@ ENTRY(pgm_check_handler)
398398
tm __LC_PGM_ILC+2,0x02 # check for transaction abort
399399
jz 2f
400400
mvc __THREAD_trap_tdb(256,%r14),0(%r13)
401-
2: aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
402-
la %r11,STACK_FRAME_OVERHEAD(%r15)
401+
2: la %r11,STACK_FRAME_OVERHEAD(%r15)
403402
stmg %r0,%r7,__PT_R0(%r11)
404403
mvc __PT_R8(64,%r11),__LC_SAVE_AREA_SYNC
405404
stmg %r8,%r9,__PT_PSW(%r11)
@@ -526,7 +525,6 @@ io_work:
526525
#
527526
io_work_user:
528527
lg %r1,__LC_KERNEL_STACK
529-
aghi %r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
530528
mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
531529
xc __SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1)
532530
la %r11,STACK_FRAME_OVERHEAD(%r1)
@@ -688,7 +686,6 @@ mcck_skip:
688686
tm __PT_PSW+1(%r11),0x01 # returning to user ?
689687
jno mcck_return
690688
lg %r1,__LC_KERNEL_STACK # switch to kernel stack
691-
aghi %r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
692689
mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
693690
xc __SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1)
694691
la %r11,STACK_FRAME_OVERHEAD(%r1)
@@ -755,14 +752,12 @@ ENTRY(restart_int_handler)
755752
* Setup a pt_regs so that show_trace can provide a good call trace.
756753
*/
757754
stack_overflow:
758-
lg %r11,__LC_PANIC_STACK # change to panic stack
759-
aghi %r11,-__PT_SIZE # create pt_regs
755+
lg %r15,__LC_PANIC_STACK # change to panic stack
756+
la %r11,STACK_FRAME_OVERHEAD(%r15)
760757
stmg %r0,%r7,__PT_R0(%r11)
761758
stmg %r8,%r9,__PT_PSW(%r11)
762759
mvc __PT_R8(64,%r11),0(%r14)
763760
stg %r10,__PT_ORIG_GPR2(%r11) # store last break to orig_gpr2
764-
lgr %r15,%r11
765-
aghi %r15,-STACK_FRAME_OVERHEAD
766761
xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
767762
lgr %r2,%r11 # pass pointer to pt_regs
768763
jg kernel_stack_overflow
@@ -846,15 +841,14 @@ cleanup_system_call:
846841
mvc __TI_last_break(8,%r12),16(%r11)
847842
0: # set up saved register r11
848843
lg %r15,__LC_KERNEL_STACK
849-
aghi %r15,-__PT_SIZE
850-
stg %r15,24(%r11) # r11 pt_regs pointer
844+
la %r9,STACK_FRAME_OVERHEAD(%r15)
845+
stg %r9,24(%r11) # r11 pt_regs pointer
851846
# fill pt_regs
852-
mvc __PT_R8(64,%r15),__LC_SAVE_AREA_SYNC
853-
stmg %r0,%r7,__PT_R0(%r15)
854-
mvc __PT_PSW(16,%r15),__LC_SVC_OLD_PSW
855-
mvc __PT_INT_CODE(4,%r15),__LC_SVC_ILC
847+
mvc __PT_R8(64,%r9),__LC_SAVE_AREA_SYNC
848+
stmg %r0,%r7,__PT_R0(%r9)
849+
mvc __PT_PSW(16,%r9),__LC_SVC_OLD_PSW
850+
mvc __PT_INT_CODE(4,%r9),__LC_SVC_ILC
856851
# setup saved register r15
857-
aghi %r15,-STACK_FRAME_OVERHEAD
858852
stg %r15,56(%r11) # r15 stack pointer
859853
# set new psw address and exit
860854
larl %r9,sysc_do_svc

‎arch/s390/kernel/setup.c

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -377,11 +377,14 @@ static void __init setup_lowcore(void)
377377
PSW_MASK_DAT | PSW_MASK_MCHECK;
378378
lc->io_new_psw.addr = PSW_ADDR_AMODE | (unsigned long) io_int_handler;
379379
lc->clock_comparator = -1ULL;
380-
lc->kernel_stack = ((unsigned long) &init_thread_union) + THREAD_SIZE;
380+
lc->kernel_stack = ((unsigned long) &init_thread_union)
381+
+ THREAD_SIZE - STACK_FRAME_OVERHEAD - sizeof(struct pt_regs);
381382
lc->async_stack = (unsigned long)
382-
__alloc_bootmem(ASYNC_SIZE, ASYNC_SIZE, 0) + ASYNC_SIZE;
383+
__alloc_bootmem(ASYNC_SIZE, ASYNC_SIZE, 0)
384+
+ ASYNC_SIZE - STACK_FRAME_OVERHEAD - sizeof(struct pt_regs);
383385
lc->panic_stack = (unsigned long)
384-
__alloc_bootmem(PAGE_SIZE, PAGE_SIZE, 0) + PAGE_SIZE;
386+
__alloc_bootmem(PAGE_SIZE, PAGE_SIZE, 0)
387+
+ PAGE_SIZE - STACK_FRAME_OVERHEAD - sizeof(struct pt_regs);
385388
lc->current_task = (unsigned long) init_thread_union.thread_info.task;
386389
lc->thread_info = (unsigned long) &init_thread_union;
387390
lc->machine_flags = S390_lowcore.machine_flags;

‎arch/s390/kernel/smp.c

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -181,8 +181,10 @@ static int __cpuinit pcpu_alloc_lowcore(struct pcpu *pcpu, int cpu)
181181
lc = pcpu->lowcore;
182182
memcpy(lc, &S390_lowcore, 512);
183183
memset((char *) lc + 512, 0, sizeof(*lc) - 512);
184-
lc->async_stack = pcpu->async_stack + ASYNC_SIZE;
185-
lc->panic_stack = pcpu->panic_stack + PAGE_SIZE;
184+
lc->async_stack = pcpu->async_stack + ASYNC_SIZE
185+
- STACK_FRAME_OVERHEAD - sizeof(struct pt_regs);
186+
lc->panic_stack = pcpu->panic_stack + PAGE_SIZE
187+
- STACK_FRAME_OVERHEAD - sizeof(struct pt_regs);
186188
lc->cpu_nr = cpu;
187189
#ifndef CONFIG_64BIT
188190
if (MACHINE_HAS_IEEE) {
@@ -253,7 +255,8 @@ static void pcpu_attach_task(struct pcpu *pcpu, struct task_struct *tsk)
253255
struct _lowcore *lc = pcpu->lowcore;
254256
struct thread_info *ti = task_thread_info(tsk);
255257

256-
lc->kernel_stack = (unsigned long) task_stack_page(tsk) + THREAD_SIZE;
258+
lc->kernel_stack = (unsigned long) task_stack_page(tsk)
259+
+ THREAD_SIZE - STACK_FRAME_OVERHEAD - sizeof(struct pt_regs);
257260
lc->thread_info = (unsigned long) task_thread_info(tsk);
258261
lc->current_task = (unsigned long) tsk;
259262
lc->user_timer = ti->user_timer;
@@ -810,8 +813,10 @@ void __init smp_prepare_boot_cpu(void)
810813
pcpu->state = CPU_STATE_CONFIGURED;
811814
pcpu->address = boot_cpu_address;
812815
pcpu->lowcore = (struct _lowcore *)(unsigned long) store_prefix();
813-
pcpu->async_stack = S390_lowcore.async_stack - ASYNC_SIZE;
814-
pcpu->panic_stack = S390_lowcore.panic_stack - PAGE_SIZE;
816+
pcpu->async_stack = S390_lowcore.async_stack - ASYNC_SIZE
817+
+ STACK_FRAME_OVERHEAD + sizeof(struct pt_regs);
818+
pcpu->panic_stack = S390_lowcore.panic_stack - PAGE_SIZE
819+
+ STACK_FRAME_OVERHEAD + sizeof(struct pt_regs);
815820
S390_lowcore.percpu_offset = __per_cpu_offset[0];
816821
smp_cpu_set_polarization(0, POLARIZATION_UNKNOWN);
817822
set_cpu_present(0, true);

0 commit comments

Comments
 (0)