Skip to content

Commit 118bd31

Browse files
author
Martin Schwidefsky
committed
s390/mm: add no-dat TLB flush optimization
Signed-off-by: Martin Schwidefsky <[email protected]>
1 parent c9b5ad5 commit 118bd31

File tree

6 files changed

+129
-42
lines changed

6 files changed

+129
-42
lines changed

‎arch/s390/include/asm/pgtable.h

Lines changed: 29 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -952,15 +952,27 @@ static inline pte_t pte_mkhuge(pte_t pte)
952952
#define IPTE_GLOBAL 0
953953
#define IPTE_LOCAL 1
954954

955-
static inline void __ptep_ipte(unsigned long address, pte_t *ptep, int local)
955+
#define IPTE_NODAT 0x400
956+
957+
static inline void __ptep_ipte(unsigned long address, pte_t *ptep,
958+
unsigned long opt, int local)
956959
{
957960
unsigned long pto = (unsigned long) ptep;
958961

959-
/* Invalidation + TLB flush for the pte */
962+
if (__builtin_constant_p(opt) && opt == 0) {
963+
/* Invalidation + TLB flush for the pte */
964+
asm volatile(
965+
" .insn rrf,0xb2210000,%[r1],%[r2],0,%[m4]"
966+
: "+m" (*ptep) : [r1] "a" (pto), [r2] "a" (address),
967+
[m4] "i" (local));
968+
return;
969+
}
970+
971+
/* Invalidate ptes with options + TLB flush of the ptes */
960972
asm volatile(
961-
" .insn rrf,0xb2210000,%[r1],%[r2],0,%[m4]"
962-
: "+m" (*ptep) : [r1] "a" (pto), [r2] "a" (address),
963-
[m4] "i" (local));
973+
" .insn rrf,0xb2210000,%[r1],%[r2],%[r3],%[m4]"
974+
: [r2] "+a" (address), [r3] "+a" (opt)
975+
: [r1] "a" (pto), [m4] "i" (local) : "memory");
964976
}
965977

966978
static inline void __ptep_ipte_range(unsigned long address, int nr,
@@ -1341,31 +1353,36 @@ static inline void __pmdp_csp(pmd_t *pmdp)
13411353
#define IDTE_GLOBAL 0
13421354
#define IDTE_LOCAL 1
13431355

1344-
static inline void __pmdp_idte(unsigned long address, pmd_t *pmdp, int local)
1356+
#define IDTE_PTOA 0x0800
1357+
#define IDTE_NODAT 0x1000
1358+
1359+
static inline void __pmdp_idte(unsigned long addr, pmd_t *pmdp,
1360+
unsigned long opt, int local)
13451361
{
13461362
unsigned long sto;
13471363

1348-
sto = (unsigned long) pmdp - pmd_index(address) * sizeof(pmd_t);
1364+
sto = (unsigned long) pmdp - pmd_index(addr) * sizeof(pmd_t);
13491365
asm volatile(
13501366
" .insn rrf,0xb98e0000,%[r1],%[r2],0,%[m4]"
13511367
: "+m" (*pmdp)
1352-
: [r1] "a" (sto), [r2] "a" ((address & HPAGE_MASK)),
1368+
: [r1] "a" (sto), [r2] "a" ((addr & HPAGE_MASK) | opt),
13531369
[m4] "i" (local)
13541370
: "cc" );
13551371
}
13561372

1357-
static inline void __pudp_idte(unsigned long address, pud_t *pudp, int local)
1373+
static inline void __pudp_idte(unsigned long addr, pud_t *pudp,
1374+
unsigned long opt, int local)
13581375
{
13591376
unsigned long r3o;
13601377

1361-
r3o = (unsigned long) pudp - pud_index(address) * sizeof(pud_t);
1378+
r3o = (unsigned long) pudp - pud_index(addr) * sizeof(pud_t);
13621379
r3o |= _ASCE_TYPE_REGION3;
13631380
asm volatile(
13641381
" .insn rrf,0xb98e0000,%[r1],%[r2],0,%[m4]"
13651382
: "+m" (*pudp)
1366-
: [r1] "a" (r3o), [r2] "a" ((address & PUD_MASK)),
1383+
: [r1] "a" (r3o), [r2] "a" ((addr & PUD_MASK) | opt),
13671384
[m4] "i" (local)
1368-
: "cc");
1385+
: "cc" );
13691386
}
13701387

13711388
pmd_t pmdp_xchg_direct(struct mm_struct *, unsigned long, pmd_t *, pmd_t);

‎arch/s390/include/asm/setup.h

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,9 @@
2929
#define MACHINE_FLAG_TE _BITUL(11)
3030
#define MACHINE_FLAG_TLB_LC _BITUL(12)
3131
#define MACHINE_FLAG_VX _BITUL(13)
32-
#define MACHINE_FLAG_NX _BITUL(14)
33-
#define MACHINE_FLAG_GS _BITUL(15)
32+
#define MACHINE_FLAG_TLB_GUEST _BITUL(14)
33+
#define MACHINE_FLAG_NX _BITUL(15)
34+
#define MACHINE_FLAG_GS _BITUL(16)
3435

3536
#define LPP_MAGIC _BITUL(31)
3637
#define LPP_PFAULT_PID_MASK _AC(0xffffffff, UL)
@@ -68,6 +69,7 @@ extern void detect_memory_memblock(void);
6869
#define MACHINE_HAS_TE (S390_lowcore.machine_flags & MACHINE_FLAG_TE)
6970
#define MACHINE_HAS_TLB_LC (S390_lowcore.machine_flags & MACHINE_FLAG_TLB_LC)
7071
#define MACHINE_HAS_VX (S390_lowcore.machine_flags & MACHINE_FLAG_VX)
72+
#define MACHINE_HAS_TLB_GUEST (S390_lowcore.machine_flags & MACHINE_FLAG_TLB_GUEST)
7173
#define MACHINE_HAS_NX (S390_lowcore.machine_flags & MACHINE_FLAG_NX)
7274
#define MACHINE_HAS_GS (S390_lowcore.machine_flags & MACHINE_FLAG_GS)
7375

‎arch/s390/include/asm/tlbflush.h

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,13 @@ static inline void __tlb_flush_local(void)
2020
*/
2121
static inline void __tlb_flush_idte(unsigned long asce)
2222
{
23+
unsigned long opt;
24+
25+
opt = IDTE_PTOA;
2326
/* Global TLB flush for the mm */
2427
asm volatile(
2528
" .insn rrf,0xb98e0000,0,%0,%1,0"
26-
: : "a" (2048), "a" (asce) : "cc");
29+
: : "a" (opt), "a" (asce) : "cc");
2730
}
2831

2932
#ifdef CONFIG_SMP

‎arch/s390/mm/pageattr.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -328,7 +328,7 @@ static void ipte_range(pte_t *pte, unsigned long address, int nr)
328328
return;
329329
}
330330
for (i = 0; i < nr; i++) {
331-
__ptep_ipte(address, pte, IPTE_GLOBAL);
331+
__ptep_ipte(address, pte, 0, IPTE_GLOBAL);
332332
address += PAGE_SIZE;
333333
pte++;
334334
}

‎arch/s390/mm/pgtable.c

Lines changed: 87 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,38 @@
2525
#include <asm/mmu_context.h>
2626
#include <asm/page-states.h>
2727

28+
static inline void ptep_ipte_local(struct mm_struct *mm, unsigned long addr,
29+
pte_t *ptep)
30+
{
31+
unsigned long opt, asce;
32+
33+
if (MACHINE_HAS_TLB_GUEST) {
34+
opt = 0;
35+
asce = READ_ONCE(mm->context.gmap_asce);
36+
if (asce == 0UL)
37+
opt |= IPTE_NODAT;
38+
__ptep_ipte(addr, ptep, opt, IPTE_LOCAL);
39+
} else {
40+
__ptep_ipte(addr, ptep, 0, IPTE_LOCAL);
41+
}
42+
}
43+
44+
static inline void ptep_ipte_global(struct mm_struct *mm, unsigned long addr,
45+
pte_t *ptep)
46+
{
47+
unsigned long opt, asce;
48+
49+
if (MACHINE_HAS_TLB_GUEST) {
50+
opt = 0;
51+
asce = READ_ONCE(mm->context.gmap_asce);
52+
if (asce == 0UL)
53+
opt |= IPTE_NODAT;
54+
__ptep_ipte(addr, ptep, opt, IPTE_GLOBAL);
55+
} else {
56+
__ptep_ipte(addr, ptep, 0, IPTE_GLOBAL);
57+
}
58+
}
59+
2860
static inline pte_t ptep_flush_direct(struct mm_struct *mm,
2961
unsigned long addr, pte_t *ptep)
3062
{
@@ -36,9 +68,9 @@ static inline pte_t ptep_flush_direct(struct mm_struct *mm,
3668
atomic_inc(&mm->context.flush_count);
3769
if (MACHINE_HAS_TLB_LC &&
3870
cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
39-
__ptep_ipte(addr, ptep, IPTE_LOCAL);
71+
ptep_ipte_local(mm, addr, ptep);
4072
else
41-
__ptep_ipte(addr, ptep, IPTE_GLOBAL);
73+
ptep_ipte_global(mm, addr, ptep);
4274
atomic_dec(&mm->context.flush_count);
4375
return old;
4476
}
@@ -57,7 +89,7 @@ static inline pte_t ptep_flush_lazy(struct mm_struct *mm,
5789
pte_val(*ptep) |= _PAGE_INVALID;
5890
mm->context.flush_mm = 1;
5991
} else
60-
__ptep_ipte(addr, ptep, IPTE_GLOBAL);
92+
ptep_ipte_global(mm, addr, ptep);
6193
atomic_dec(&mm->context.flush_count);
6294
return old;
6395
}
@@ -290,6 +322,26 @@ void ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr,
290322
}
291323
EXPORT_SYMBOL(ptep_modify_prot_commit);
292324

325+
static inline void pmdp_idte_local(struct mm_struct *mm,
326+
unsigned long addr, pmd_t *pmdp)
327+
{
328+
if (MACHINE_HAS_TLB_GUEST)
329+
__pmdp_idte(addr, pmdp, IDTE_NODAT, IDTE_LOCAL);
330+
else
331+
__pmdp_idte(addr, pmdp, 0, IDTE_LOCAL);
332+
}
333+
334+
static inline void pmdp_idte_global(struct mm_struct *mm,
335+
unsigned long addr, pmd_t *pmdp)
336+
{
337+
if (MACHINE_HAS_TLB_GUEST)
338+
__pmdp_idte(addr, pmdp, IDTE_NODAT, IDTE_GLOBAL);
339+
else if (MACHINE_HAS_IDTE)
340+
__pmdp_idte(addr, pmdp, 0, IDTE_GLOBAL);
341+
else
342+
__pmdp_csp(pmdp);
343+
}
344+
293345
static inline pmd_t pmdp_flush_direct(struct mm_struct *mm,
294346
unsigned long addr, pmd_t *pmdp)
295347
{
@@ -298,16 +350,12 @@ static inline pmd_t pmdp_flush_direct(struct mm_struct *mm,
298350
old = *pmdp;
299351
if (pmd_val(old) & _SEGMENT_ENTRY_INVALID)
300352
return old;
301-
if (!MACHINE_HAS_IDTE) {
302-
__pmdp_csp(pmdp);
303-
return old;
304-
}
305353
atomic_inc(&mm->context.flush_count);
306354
if (MACHINE_HAS_TLB_LC &&
307355
cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
308-
__pmdp_idte(addr, pmdp, IDTE_LOCAL);
356+
pmdp_idte_local(mm, addr, pmdp);
309357
else
310-
__pmdp_idte(addr, pmdp, IDTE_GLOBAL);
358+
pmdp_idte_global(mm, addr, pmdp);
311359
atomic_dec(&mm->context.flush_count);
312360
return old;
313361
}
@@ -325,10 +373,9 @@ static inline pmd_t pmdp_flush_lazy(struct mm_struct *mm,
325373
cpumask_of(smp_processor_id()))) {
326374
pmd_val(*pmdp) |= _SEGMENT_ENTRY_INVALID;
327375
mm->context.flush_mm = 1;
328-
} else if (MACHINE_HAS_IDTE)
329-
__pmdp_idte(addr, pmdp, IDTE_GLOBAL);
330-
else
331-
__pmdp_csp(pmdp);
376+
} else {
377+
pmdp_idte_global(mm, addr, pmdp);
378+
}
332379
atomic_dec(&mm->context.flush_count);
333380
return old;
334381
}
@@ -359,28 +406,44 @@ pmd_t pmdp_xchg_lazy(struct mm_struct *mm, unsigned long addr,
359406
}
360407
EXPORT_SYMBOL(pmdp_xchg_lazy);
361408

362-
static inline pud_t pudp_flush_direct(struct mm_struct *mm,
363-
unsigned long addr, pud_t *pudp)
409+
static inline void pudp_idte_local(struct mm_struct *mm,
410+
unsigned long addr, pud_t *pudp)
364411
{
365-
pud_t old;
412+
if (MACHINE_HAS_TLB_GUEST)
413+
__pudp_idte(addr, pudp, IDTE_NODAT, IDTE_LOCAL);
414+
else
415+
__pudp_idte(addr, pudp, 0, IDTE_LOCAL);
416+
}
366417

367-
old = *pudp;
368-
if (pud_val(old) & _REGION_ENTRY_INVALID)
369-
return old;
370-
if (!MACHINE_HAS_IDTE) {
418+
static inline void pudp_idte_global(struct mm_struct *mm,
419+
unsigned long addr, pud_t *pudp)
420+
{
421+
if (MACHINE_HAS_TLB_GUEST)
422+
__pudp_idte(addr, pudp, IDTE_NODAT, IDTE_GLOBAL);
423+
else if (MACHINE_HAS_IDTE)
424+
__pudp_idte(addr, pudp, 0, IDTE_GLOBAL);
425+
else
371426
/*
372427
* Invalid bit position is the same for pmd and pud, so we can
373428
* re-use _pmd_csp() here
374429
*/
375430
__pmdp_csp((pmd_t *) pudp);
431+
}
432+
433+
static inline pud_t pudp_flush_direct(struct mm_struct *mm,
434+
unsigned long addr, pud_t *pudp)
435+
{
436+
pud_t old;
437+
438+
old = *pudp;
439+
if (pud_val(old) & _REGION_ENTRY_INVALID)
376440
return old;
377-
}
378441
atomic_inc(&mm->context.flush_count);
379442
if (MACHINE_HAS_TLB_LC &&
380443
cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
381-
__pudp_idte(addr, pudp, IDTE_LOCAL);
444+
pudp_idte_local(mm, addr, pudp);
382445
else
383-
__pudp_idte(addr, pudp, IDTE_GLOBAL);
446+
pudp_idte_global(mm, addr, pudp);
384447
atomic_dec(&mm->context.flush_count);
385448
return old;
386449
}
@@ -645,7 +708,7 @@ bool test_and_clear_guest_dirty(struct mm_struct *mm, unsigned long addr)
645708
pte = *ptep;
646709
if (dirty && (pte_val(pte) & _PAGE_PRESENT)) {
647710
pgste = pgste_pte_notify(mm, addr, ptep, pgste);
648-
__ptep_ipte(addr, ptep, IPTE_GLOBAL);
711+
ptep_ipte_global(mm, addr, ptep);
649712
if (MACHINE_HAS_ESOP || !(pte_val(pte) & _PAGE_WRITE))
650713
pte_val(pte) |= _PAGE_PROTECT;
651714
else

‎drivers/s390/char/sclp_early.c

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ struct read_info_sccb {
3939
u8 fac84; /* 84 */
4040
u8 fac85; /* 85 */
4141
u8 _pad_86[91 - 86]; /* 86-90 */
42-
u8 flags; /* 91 */
42+
u8 fac91; /* 91 */
4343
u8 _pad_92[98 - 92]; /* 92-97 */
4444
u8 fac98; /* 98 */
4545
u8 hamaxpow; /* 99 */
@@ -103,6 +103,8 @@ static void __init sclp_early_facilities_detect(struct read_info_sccb *sccb)
103103
sclp.has_kss = !!(sccb->fac98 & 0x01);
104104
if (sccb->fac85 & 0x02)
105105
S390_lowcore.machine_flags |= MACHINE_FLAG_ESOP;
106+
if (sccb->fac91 & 0x40)
107+
S390_lowcore.machine_flags |= MACHINE_FLAG_TLB_GUEST;
106108
sclp.rnmax = sccb->rnmax ? sccb->rnmax : sccb->rnmax2;
107109
sclp.rzm = sccb->rnsize ? sccb->rnsize : sccb->rnsize2;
108110
sclp.rzm <<= 20;
@@ -139,7 +141,7 @@ static void __init sclp_early_facilities_detect(struct read_info_sccb *sccb)
139141

140142
/* Save IPL information */
141143
sclp_ipl_info.is_valid = 1;
142-
if (sccb->flags & 0x2)
144+
if (sccb->fac91 & 0x2)
143145
sclp_ipl_info.has_dump = 1;
144146
memcpy(&sclp_ipl_info.loadparm, &sccb->loadparm, LOADPARM_LEN);
145147

0 commit comments

Comments
 (0)