Skip to content

Commit 28c807e

Browse files
author
Martin Schwidefsky
committed
s390/mm: add guest ASCE TLB flush optimization
Signed-off-by: Martin Schwidefsky <[email protected]>
1 parent 118bd31 commit 28c807e

File tree

4 files changed

+70
-28
lines changed

4 files changed

+70
-28
lines changed

‎arch/s390/include/asm/pgtable.h

Lines changed: 43 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -953,9 +953,11 @@ static inline pte_t pte_mkhuge(pte_t pte)
953953
#define IPTE_LOCAL 1
954954

955955
#define IPTE_NODAT 0x400
956+
#define IPTE_GUEST_ASCE 0x800
956957

957958
static inline void __ptep_ipte(unsigned long address, pte_t *ptep,
958-
unsigned long opt, int local)
959+
unsigned long opt, unsigned long asce,
960+
int local)
959961
{
960962
unsigned long pto = (unsigned long) ptep;
961963

@@ -969,6 +971,7 @@ static inline void __ptep_ipte(unsigned long address, pte_t *ptep,
969971
}
970972

971973
/* Invalidate ptes with options + TLB flush of the ptes */
974+
opt = opt | (asce & _ASCE_ORIGIN);
972975
asm volatile(
973976
" .insn rrf,0xb2210000,%[r1],%[r2],%[r3],%[m4]"
974977
: [r2] "+a" (address), [r3] "+a" (opt)
@@ -1355,34 +1358,59 @@ static inline void __pmdp_csp(pmd_t *pmdp)
13551358

13561359
#define IDTE_PTOA 0x0800
13571360
#define IDTE_NODAT 0x1000
1361+
#define IDTE_GUEST_ASCE 0x2000
13581362

13591363
static inline void __pmdp_idte(unsigned long addr, pmd_t *pmdp,
1360-
unsigned long opt, int local)
1364+
unsigned long opt, unsigned long asce,
1365+
int local)
13611366
{
13621367
unsigned long sto;
13631368

13641369
sto = (unsigned long) pmdp - pmd_index(addr) * sizeof(pmd_t);
1365-
asm volatile(
1366-
" .insn rrf,0xb98e0000,%[r1],%[r2],0,%[m4]"
1367-
: "+m" (*pmdp)
1368-
: [r1] "a" (sto), [r2] "a" ((addr & HPAGE_MASK) | opt),
1369-
[m4] "i" (local)
1370-
: "cc" );
1370+
if (__builtin_constant_p(opt) && opt == 0) {
1371+
/* flush without guest asce */
1372+
asm volatile(
1373+
" .insn rrf,0xb98e0000,%[r1],%[r2],0,%[m4]"
1374+
: "+m" (*pmdp)
1375+
: [r1] "a" (sto), [r2] "a" ((addr & HPAGE_MASK)),
1376+
[m4] "i" (local)
1377+
: "cc" );
1378+
} else {
1379+
/* flush with guest asce */
1380+
asm volatile(
1381+
" .insn rrf,0xb98e0000,%[r1],%[r2],%[r3],%[m4]"
1382+
: "+m" (*pmdp)
1383+
: [r1] "a" (sto), [r2] "a" ((addr & HPAGE_MASK) | opt),
1384+
[r3] "a" (asce), [m4] "i" (local)
1385+
: "cc" );
1386+
}
13711387
}
13721388

13731389
static inline void __pudp_idte(unsigned long addr, pud_t *pudp,
1374-
unsigned long opt, int local)
1390+
unsigned long opt, unsigned long asce,
1391+
int local)
13751392
{
13761393
unsigned long r3o;
13771394

13781395
r3o = (unsigned long) pudp - pud_index(addr) * sizeof(pud_t);
13791396
r3o |= _ASCE_TYPE_REGION3;
1380-
asm volatile(
1381-
" .insn rrf,0xb98e0000,%[r1],%[r2],0,%[m4]"
1382-
: "+m" (*pudp)
1383-
: [r1] "a" (r3o), [r2] "a" ((addr & PUD_MASK) | opt),
1384-
[m4] "i" (local)
1385-
: "cc" );
1397+
if (__builtin_constant_p(opt) && opt == 0) {
1398+
/* flush without guest asce */
1399+
asm volatile(
1400+
" .insn rrf,0xb98e0000,%[r1],%[r2],0,%[m4]"
1401+
: "+m" (*pudp)
1402+
: [r1] "a" (r3o), [r2] "a" ((addr & PUD_MASK)),
1403+
[m4] "i" (local)
1404+
: "cc");
1405+
} else {
1406+
/* flush with guest asce */
1407+
asm volatile(
1408+
" .insn rrf,0xb98e0000,%[r1],%[r2],%[r3],%[m4]"
1409+
: "+m" (*pudp)
1410+
: [r1] "a" (r3o), [r2] "a" ((addr & PUD_MASK) | opt),
1411+
[r3] "a" (asce), [m4] "i" (local)
1412+
: "cc" );
1413+
}
13861414
}
13871415

13881416
pmd_t pmdp_xchg_direct(struct mm_struct *, unsigned long, pmd_t *, pmd_t);

‎arch/s390/include/asm/tlbflush.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@ static inline void __tlb_flush_idte(unsigned long asce)
2323
unsigned long opt;
2424

2525
opt = IDTE_PTOA;
26+
if (MACHINE_HAS_TLB_GUEST)
27+
opt |= IDTE_GUEST_ASCE;
2628
/* Global TLB flush for the mm */
2729
asm volatile(
2830
" .insn rrf,0xb98e0000,0,%0,%1,0"

‎arch/s390/mm/pageattr.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -328,7 +328,7 @@ static void ipte_range(pte_t *pte, unsigned long address, int nr)
328328
return;
329329
}
330330
for (i = 0; i < nr; i++) {
331-
__ptep_ipte(address, pte, 0, IPTE_GLOBAL);
331+
__ptep_ipte(address, pte, 0, 0, IPTE_GLOBAL);
332332
address += PAGE_SIZE;
333333
pte++;
334334
}

‎arch/s390/mm/pgtable.c

Lines changed: 24 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,13 @@ static inline void ptep_ipte_local(struct mm_struct *mm, unsigned long addr,
3535
asce = READ_ONCE(mm->context.gmap_asce);
3636
if (asce == 0UL)
3737
opt |= IPTE_NODAT;
38-
__ptep_ipte(addr, ptep, opt, IPTE_LOCAL);
38+
if (asce != -1UL) {
39+
asce = asce ? : mm->context.asce;
40+
opt |= IPTE_GUEST_ASCE;
41+
}
42+
__ptep_ipte(addr, ptep, opt, asce, IPTE_LOCAL);
3943
} else {
40-
__ptep_ipte(addr, ptep, 0, IPTE_LOCAL);
44+
__ptep_ipte(addr, ptep, 0, 0, IPTE_LOCAL);
4145
}
4246
}
4347

@@ -51,9 +55,13 @@ static inline void ptep_ipte_global(struct mm_struct *mm, unsigned long addr,
5155
asce = READ_ONCE(mm->context.gmap_asce);
5256
if (asce == 0UL)
5357
opt |= IPTE_NODAT;
54-
__ptep_ipte(addr, ptep, opt, IPTE_GLOBAL);
58+
if (asce != -1UL) {
59+
asce = asce ? : mm->context.asce;
60+
opt |= IPTE_GUEST_ASCE;
61+
}
62+
__ptep_ipte(addr, ptep, opt, asce, IPTE_GLOBAL);
5563
} else {
56-
__ptep_ipte(addr, ptep, 0, IPTE_GLOBAL);
64+
__ptep_ipte(addr, ptep, 0, 0, IPTE_GLOBAL);
5765
}
5866
}
5967

@@ -326,18 +334,20 @@ static inline void pmdp_idte_local(struct mm_struct *mm,
326334
unsigned long addr, pmd_t *pmdp)
327335
{
328336
if (MACHINE_HAS_TLB_GUEST)
329-
__pmdp_idte(addr, pmdp, IDTE_NODAT, IDTE_LOCAL);
337+
__pmdp_idte(addr, pmdp, IDTE_NODAT | IDTE_GUEST_ASCE,
338+
mm->context.asce, IDTE_LOCAL);
330339
else
331-
__pmdp_idte(addr, pmdp, 0, IDTE_LOCAL);
340+
__pmdp_idte(addr, pmdp, 0, 0, IDTE_LOCAL);
332341
}
333342

334343
static inline void pmdp_idte_global(struct mm_struct *mm,
335344
unsigned long addr, pmd_t *pmdp)
336345
{
337346
if (MACHINE_HAS_TLB_GUEST)
338-
__pmdp_idte(addr, pmdp, IDTE_NODAT, IDTE_GLOBAL);
347+
__pmdp_idte(addr, pmdp, IDTE_NODAT | IDTE_GUEST_ASCE,
348+
mm->context.asce, IDTE_GLOBAL);
339349
else if (MACHINE_HAS_IDTE)
340-
__pmdp_idte(addr, pmdp, 0, IDTE_GLOBAL);
350+
__pmdp_idte(addr, pmdp, 0, 0, IDTE_GLOBAL);
341351
else
342352
__pmdp_csp(pmdp);
343353
}
@@ -410,18 +420,20 @@ static inline void pudp_idte_local(struct mm_struct *mm,
410420
unsigned long addr, pud_t *pudp)
411421
{
412422
if (MACHINE_HAS_TLB_GUEST)
413-
__pudp_idte(addr, pudp, IDTE_NODAT, IDTE_LOCAL);
423+
__pudp_idte(addr, pudp, IDTE_NODAT | IDTE_GUEST_ASCE,
424+
mm->context.asce, IDTE_LOCAL);
414425
else
415-
__pudp_idte(addr, pudp, 0, IDTE_LOCAL);
426+
__pudp_idte(addr, pudp, 0, 0, IDTE_LOCAL);
416427
}
417428

418429
static inline void pudp_idte_global(struct mm_struct *mm,
419430
unsigned long addr, pud_t *pudp)
420431
{
421432
if (MACHINE_HAS_TLB_GUEST)
422-
__pudp_idte(addr, pudp, IDTE_NODAT, IDTE_GLOBAL);
433+
__pudp_idte(addr, pudp, IDTE_NODAT | IDTE_GUEST_ASCE,
434+
mm->context.asce, IDTE_GLOBAL);
423435
else if (MACHINE_HAS_IDTE)
424-
__pudp_idte(addr, pudp, 0, IDTE_GLOBAL);
436+
__pudp_idte(addr, pudp, 0, 0, IDTE_GLOBAL);
425437
else
426438
/*
427439
* Invalid bit position is the same for pmd and pud, so we can

0 commit comments

Comments
 (0)