mm: cleanup *pte_alloc* interfaces

kiryl · torvalds · commit 3ed3a4f0ddff · 2016-03-17T15:09:34.000-07:00
There are few things about *pte_alloc*() helpers worth cleaning up:

 - 'vma' argument is unused, let's drop it;

 - most __pte_alloc() callers do speculative check for pmd_none(),
   before taking ptl: let's introduce pte_alloc() macro which does
   the check.

   The only direct user of __pte_alloc left is userfaultfd, which has
   different expectation about atomicity wrt pmd.

 - pte_alloc_map() and pte_alloc_map_lock() are redefined using
   pte_alloc().

[sudeep.holla@arm.com: fix build for arm64 hugetlbpage]
[sfr@canb.auug.org.au: fix arch/arm/mm/mmu.c some more]
Signed-off-by: Kirill A. Shutemov &lt;kirill.shutemov@linux.intel.com&gt;
Cc: Dave Hansen &lt;dave.hansen@intel.com&gt;
Signed-off-by: Sudeep Holla &lt;sudeep.holla@arm.com&gt;
Acked-by: Kirill A. Shutemov &lt;kirill.shutemov@linux.intel.com&gt;
Signed-off-by: Stephen Rothwell &lt;sfr@canb.auug.org.au&gt;
Signed-off-by: Andrew Morton &lt;akpm@linux-foundation.org&gt;
Signed-off-by: Linus Torvalds &lt;torvalds@linux-foundation.org&gt;
diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c
@@ -732,7 +732,7 @@ static void *__init late_alloc(unsigned long sz)
 	return ptr;
 }
 
-static pte_t * __init pte_alloc(pmd_t *pmd, unsigned long addr,
+static pte_t * __init arm_pte_alloc(pmd_t *pmd, unsigned long addr,
 				unsigned long prot,
 				void *(*alloc)(unsigned long sz))
 {
@@ -747,7 +747,7 @@ static pte_t * __init pte_alloc(pmd_t *pmd, unsigned long addr,
 static pte_t * __init early_pte_alloc(pmd_t *pmd, unsigned long addr,
 				      unsigned long prot)
 {
-	return pte_alloc(pmd, addr, prot, early_alloc);
+	return arm_pte_alloc(pmd, addr, prot, early_alloc);
 }
 
 static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr,
@@ -756,7 +756,7 @@ static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr,
 				  void *(*alloc)(unsigned long sz),
 				  bool ng)
 {
-	pte_t *pte = pte_alloc(pmd, addr, type->prot_l1, alloc);
+	pte_t *pte = arm_pte_alloc(pmd, addr, type->prot_l1, alloc);
 	do {
 		set_pte_ext(pte, pfn_pte(pfn, __pgprot(type->prot_pte)),
 			    ng ? PTE_EXT_NG : 0);
diff --git a/arch/arm/mm/pgd.c b/arch/arm/mm/pgd.c
@@ -80,7 +80,7 @@ pgd_t *pgd_alloc(struct mm_struct *mm)
 		if (!new_pmd)
 			goto no_pmd;
 
-		new_pte = pte_alloc_map(mm, NULL, new_pmd, 0);
+		new_pte = pte_alloc_map(mm, new_pmd, 0);
 		if (!new_pte)
 			goto no_pte;
 
diff --git a/arch/arm64/mm/hugetlbpage.c b/arch/arm64/mm/hugetlbpage.c
@@ -124,7 +124,7 @@ pte_t *huge_pte_alloc(struct mm_struct *mm,
 		 * will be no pte_unmap() to correspond with this
 		 * pte_alloc_map().
 		 */
-		pte = pte_alloc_map(mm, NULL, pmd, addr);
+		pte = pte_alloc_map(mm, pmd, addr);
 	} else if (sz == PMD_SIZE) {
 		if (IS_ENABLED(CONFIG_ARCH_WANT_HUGE_PMD_SHARE) &&
 		    pud_none(*pud))
diff --git a/arch/ia64/mm/hugetlbpage.c b/arch/ia64/mm/hugetlbpage.c
@@ -38,7 +38,7 @@ huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz)
 	if (pud) {
 		pmd = pmd_alloc(mm, pud, taddr);
 		if (pmd)
-			pte = pte_alloc_map(mm, NULL, pmd, taddr);
+			pte = pte_alloc_map(mm, pmd, taddr);
 	}
 	return pte;
 }
diff --git a/arch/metag/mm/hugetlbpage.c b/arch/metag/mm/hugetlbpage.c
@@ -67,7 +67,7 @@ pte_t *huge_pte_alloc(struct mm_struct *mm,
 	pgd = pgd_offset(mm, addr);
 	pud = pud_offset(pgd, addr);
 	pmd = pmd_offset(pud, addr);
-	pte = pte_alloc_map(mm, NULL, pmd, addr);
+	pte = pte_alloc_map(mm, pmd, addr);
 	pgd->pgd &= ~_PAGE_SZ_MASK;
 	pgd->pgd |= _PAGE_SZHUGE;
 
diff --git a/arch/parisc/mm/hugetlbpage.c b/arch/parisc/mm/hugetlbpage.c
@@ -63,7 +63,7 @@ pte_t *huge_pte_alloc(struct mm_struct *mm,
 	if (pud) {
 		pmd = pmd_alloc(mm, pud, addr);
 		if (pmd)
-			pte = pte_alloc_map(mm, NULL, pmd, addr);
+			pte = pte_alloc_map(mm, pmd, addr);
 	}
 	return pte;
 }
diff --git a/arch/sh/mm/hugetlbpage.c b/arch/sh/mm/hugetlbpage.c
@@ -35,7 +35,7 @@ pte_t *huge_pte_alloc(struct mm_struct *mm,
 		if (pud) {
 			pmd = pmd_alloc(mm, pud, addr);
 			if (pmd)
-				pte = pte_alloc_map(mm, NULL, pmd, addr);
+				pte = pte_alloc_map(mm, pmd, addr);
 		}
 	}
 
diff --git a/arch/sparc/mm/hugetlbpage.c b/arch/sparc/mm/hugetlbpage.c
@@ -146,7 +146,7 @@ pte_t *huge_pte_alloc(struct mm_struct *mm,
 	if (pud) {
 		pmd = pmd_alloc(mm, pud, addr);
 		if (pmd)
-			pte = pte_alloc_map(mm, NULL, pmd, addr);
+			pte = pte_alloc_map(mm, pmd, addr);
 	}
 	return pte;
 }
diff --git a/arch/tile/mm/hugetlbpage.c b/arch/tile/mm/hugetlbpage.c
@@ -77,7 +77,7 @@ pte_t *huge_pte_alloc(struct mm_struct *mm,
 		else {
 			if (sz != PAGE_SIZE << huge_shift[HUGE_SHIFT_PAGE])
 				panic("Unexpected page size %#lx\n", sz);
-			return pte_alloc_map(mm, NULL, pmd, addr);
+			return pte_alloc_map(mm, pmd, addr);
 		}
 	}
 #else
diff --git a/arch/um/kernel/skas/mmu.c b/arch/um/kernel/skas/mmu.c
@@ -31,7 +31,7 @@ static int init_stub_pte(struct mm_struct *mm, unsigned long proc,
 	if (!pmd)
 		goto out_pmd;
 
-	pte = pte_alloc_map(mm, NULL, pmd, proc);
+	pte = pte_alloc_map(mm, pmd, proc);
 	if (!pte)
 		goto out_pte;
 
diff --git a/arch/unicore32/mm/pgd.c b/arch/unicore32/mm/pgd.c
@@ -54,7 +54,7 @@ pgd_t *get_pgd_slow(struct mm_struct *mm)
 		if (!new_pmd)
 			goto no_pmd;
 
-		new_pte = pte_alloc_map(mm, NULL, new_pmd, 0);
+		new_pte = pte_alloc_map(mm, new_pmd, 0);
 		if (!new_pte)
 			goto no_pte;
 
diff --git a/arch/x86/kernel/tboot.c b/arch/x86/kernel/tboot.c
@@ -135,7 +135,7 @@ static int map_tboot_page(unsigned long vaddr, unsigned long pfn,
 	pmd = pmd_alloc(&tboot_mm, pud, vaddr);
 	if (!pmd)
 		return -1;
-	pte = pte_alloc_map(&tboot_mm, NULL, pmd, vaddr);
+	pte = pte_alloc_map(&tboot_mm, pmd, vaddr);
 	if (!pte)
 		return -1;
 	set_pte_at(&tboot_mm, vaddr, pte, pfn_pte(pfn, prot));
diff --git a/include/linux/mm.h b/include/linux/mm.h
@@ -1545,8 +1545,7 @@ static inline void mm_dec_nr_pmds(struct mm_struct *mm)
 }
 #endif
 
-int __pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
-		pmd_t *pmd, unsigned long address);
+int __pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address);
 int __pte_alloc_kernel(pmd_t *pmd, unsigned long address);
 
 /*
@@ -1672,15 +1671,15 @@ static inline void pgtable_page_dtor(struct page *page)
 	pte_unmap(pte);					\
 } while (0)
 
-#define pte_alloc_map(mm, vma, pmd, address)				\
-	((unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, vma,	\
-							pmd, address))?	\
-	 NULL: pte_offset_map(pmd, address))
+#define pte_alloc(mm, pmd, address)			\
+	(unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, pmd, address))
+
+#define pte_alloc_map(mm, pmd, address)			\
+	(pte_alloc(mm, pmd, address) ? NULL : pte_offset_map(pmd, address))
 
 #define pte_alloc_map_lock(mm, pmd, address, ptlp)	\
-	((unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, NULL,	\
-							pmd, address))?	\
-		NULL: pte_offset_map_lock(mm, pmd, address, ptlp))
+	(pte_alloc(mm, pmd, address) ?			\
+		 NULL : pte_offset_map_lock(mm, pmd, address, ptlp))
 
 #define pte_alloc_kernel(pmd, address)			\
 	((unlikely(pmd_none(*(pmd))) && __pte_alloc_kernel(pmd, address))? \
diff --git a/mm/memory.c b/mm/memory.c
@@ -562,8 +562,7 @@ void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma,
 	}
 }
 
-int __pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
-		pmd_t *pmd, unsigned long address)
+int __pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address)
 {
 	spinlock_t *ptl;
 	pgtable_t new = pte_alloc_one(mm, address);
@@ -3419,12 +3418,11 @@ static int __handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma,
 	}
 
 	/*
-	 * Use __pte_alloc instead of pte_alloc_map, because we can't
+	 * Use pte_alloc() instead of pte_alloc_map, because we can't
 	 * run pte_offset_map on the pmd, if an huge pmd could
 	 * materialize from under us from a different thread.
 	 */
-	if (unlikely(pmd_none(*pmd)) &&
-	    unlikely(__pte_alloc(mm, vma, pmd, address)))
+	if (unlikely(pte_alloc(mm, pmd, address)))
 		return VM_FAULT_OOM;
 	/*
 	 * If a huge pmd materialized under us just retry later.  Use
diff --git a/mm/mremap.c b/mm/mremap.c
@@ -213,8 +213,7 @@ unsigned long move_page_tables(struct vm_area_struct *vma,
 				continue;
 			VM_BUG_ON(pmd_trans_huge(*old_pmd));
 		}
-		if (pmd_none(*new_pmd) && __pte_alloc(new_vma->vm_mm, new_vma,
-						      new_pmd, new_addr))
+		if (pte_alloc(new_vma->vm_mm, new_pmd, new_addr))
 			break;
 		next = (new_addr + PMD_SIZE) & PMD_MASK;
 		if (extent > next - new_addr)
diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c
@@ -230,8 +230,7 @@ static __always_inline ssize_t __mcopy_atomic(struct mm_struct *dst_mm,
 			break;
 		}
 		if (unlikely(pmd_none(dst_pmdval)) &&
-		    unlikely(__pte_alloc(dst_mm, dst_vma, dst_pmd,
-					 dst_addr))) {
+		    unlikely(__pte_alloc(dst_mm, dst_pmd, dst_addr))) {
 			err = -ENOMEM;
 			break;
 		}

Original file line number	Diff line number	Diff line change
`@@ -732,7 +732,7 @@ static void *__init late_alloc(unsigned long sz)`
`732`	`732`	`return ptr;`
`733`	`733`	`}`
`734`	`734`
`735`		`-static pte_t * __init pte_alloc(pmd_t *pmd, unsigned long addr,`
	`735`	`+static pte_t * __init arm_pte_alloc(pmd_t *pmd, unsigned long addr,`
`736`	`736`	`unsigned long prot,`
`737`	`737`	`void (alloc)(unsigned long sz))`
`738`	`738`	`{`
`@@ -747,7 +747,7 @@ static pte_t * __init pte_alloc(pmd_t *pmd, unsigned long addr,`
`747`	`747`	`static pte_t * __init early_pte_alloc(pmd_t *pmd, unsigned long addr,`
`748`	`748`	`unsigned long prot)`
`749`	`749`	`{`
`750`		`- return pte_alloc(pmd, addr, prot, early_alloc);`
	`750`	`+ return arm_pte_alloc(pmd, addr, prot, early_alloc);`
`751`	`751`	`}`
`752`	`752`
`753`	`753`	`static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr,`
`@@ -756,7 +756,7 @@ static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr,`
`756`	`756`	`void (alloc)(unsigned long sz),`
`757`	`757`	`bool ng)`
`758`	`758`	`{`
`759`		`- pte_t *pte = pte_alloc(pmd, addr, type->prot_l1, alloc);`
	`759`	`+ pte_t *pte = arm_pte_alloc(pmd, addr, type->prot_l1, alloc);`
`760`	`760`	`do {`
`761`	`761`	`set_pte_ext(pte, pfn_pte(pfn, __pgprot(type->prot_pte)),`
`762`	`762`	`ng ? PTE_EXT_NG : 0);`
Original file line number	Diff line number	Diff line change
`@@ -38,7 +38,7 @@ huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz)`
`38`	`38`	`if (pud) {`
`39`	`39`	`pmd = pmd_alloc(mm, pud, taddr);`
`40`	`40`	`if (pmd)`
`41`		`- pte = pte_alloc_map(mm, NULL, pmd, taddr);`
	`41`	`+ pte = pte_alloc_map(mm, pmd, taddr);`
`42`	`42`	`}`
`43`	`43`	`return pte;`
`44`	`44`	`}`
Original file line number	Diff line number	Diff line change
`@@ -63,7 +63,7 @@ pte_t huge_pte_alloc(struct mm_struct mm,`
`63`	`63`	`if (pud) {`
`64`	`64`	`pmd = pmd_alloc(mm, pud, addr);`
`65`	`65`	`if (pmd)`
`66`		`- pte = pte_alloc_map(mm, NULL, pmd, addr);`
	`66`	`+ pte = pte_alloc_map(mm, pmd, addr);`
`67`	`67`	`}`
`68`	`68`	`return pte;`
`69`	`69`	`}`
Original file line number	Diff line number	Diff line change
`@@ -35,7 +35,7 @@ pte_t huge_pte_alloc(struct mm_struct mm,`
`35`	`35`	`if (pud) {`
`36`	`36`	`pmd = pmd_alloc(mm, pud, addr);`
`37`	`37`	`if (pmd)`
`38`		`- pte = pte_alloc_map(mm, NULL, pmd, addr);`
	`38`	`+ pte = pte_alloc_map(mm, pmd, addr);`
`39`	`39`	`}`
`40`	`40`	`}`
`41`	`41`
Original file line number	Diff line number	Diff line change
`@@ -146,7 +146,7 @@ pte_t huge_pte_alloc(struct mm_struct mm,`
`146`	`146`	`if (pud) {`
`147`	`147`	`pmd = pmd_alloc(mm, pud, addr);`
`148`	`148`	`if (pmd)`
`149`		`- pte = pte_alloc_map(mm, NULL, pmd, addr);`
	`149`	`+ pte = pte_alloc_map(mm, pmd, addr);`
`150`	`150`	`}`
`151`	`151`	`return pte;`
`152`	`152`	`}`
Original file line number	Diff line number	Diff line change
`@@ -77,7 +77,7 @@ pte_t huge_pte_alloc(struct mm_struct mm,`
`77`	`77`	`else {`
`78`	`78`	`if (sz != PAGE_SIZE << huge_shift[HUGE_SHIFT_PAGE])`
`79`	`79`	`panic("Unexpected page size %#lx\n", sz);`
`80`		`- return pte_alloc_map(mm, NULL, pmd, addr);`
	`80`	`+ return pte_alloc_map(mm, pmd, addr);`
`81`	`81`	`}`
`82`	`82`	`}`
`83`	`83`	`#else`
Original file line number	Diff line number	Diff line change
`@@ -562,8 +562,7 @@ void free_pgtables(struct mmu_gather tlb, struct vm_area_struct vma,`
`562`	`562`	`}`
`563`	`563`	`}`
`564`	`564`
`565`		`-int __pte_alloc(struct mm_struct mm, struct vm_area_struct vma,`
`566`		`- pmd_t *pmd, unsigned long address)`
	`565`	`+int __pte_alloc(struct mm_struct mm, pmd_t pmd, unsigned long address)`
`567`	`566`	`{`
`568`	`567`	`spinlock_t *ptl;`
`569`	`568`	`pgtable_t new = pte_alloc_one(mm, address);`
`@@ -3419,12 +3418,11 @@ static int __handle_mm_fault(struct mm_struct mm, struct vm_area_struct vma,`
`3419`	`3418`	`}`
`3420`	`3419`
`3421`	`3420`	`/*`
`3422`		`- * Use __pte_alloc instead of pte_alloc_map, because we can't`
	`3421`	`+ * Use pte_alloc() instead of pte_alloc_map, because we can't`
`3423`	`3422`	`* run pte_offset_map on the pmd, if an huge pmd could`
`3424`	`3423`	`* materialize from under us from a different thread.`
`3425`	`3424`	`*/`
`3426`		`- if (unlikely(pmd_none(*pmd)) &&`
`3427`		`- unlikely(__pte_alloc(mm, vma, pmd, address)))`
	`3425`	`+ if (unlikely(pte_alloc(mm, pmd, address)))`
`3428`	`3426`	`return VM_FAULT_OOM;`
`3429`	`3427`	`/*`
`3430`	`3428`	`* If a huge pmd materialized under us just retry later. Use`
Original file line number	Diff line number	Diff line change
`@@ -230,8 +230,7 @@ static __always_inline ssize_t __mcopy_atomic(struct mm_struct *dst_mm,`
`230`	`230`	`break;`
`231`	`231`	`}`
`232`	`232`	`if (unlikely(pmd_none(dst_pmdval)) &&`
`233`		`- unlikely(__pte_alloc(dst_mm, dst_vma, dst_pmd,`
`234`		`- dst_addr))) {`
	`233`	`+ unlikely(__pte_alloc(dst_mm, dst_pmd, dst_addr))) {`
`235`	`234`	`err = -ENOMEM;`
`236`	`235`	`break;`
`237`	`236`	`}`