@@ -3486,6 +3486,29 @@ static noinline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
34863486}
34873487ALLOW_ERROR_INJECTION (should_fail_alloc_page , TRUE);
34883488
3489+ static inline long __zone_watermark_unusable_free (struct zone * z ,
3490+ unsigned int order , unsigned int alloc_flags )
3491+ {
3492+ const bool alloc_harder = (alloc_flags & (ALLOC_HARDER |ALLOC_OOM ));
3493+ long unusable_free = (1 << order ) - 1 ;
3494+
3495+ /*
3496+ * If the caller does not have rights to ALLOC_HARDER then subtract
3497+ * the high-atomic reserves. This will over-estimate the size of the
3498+ * atomic reserve but it avoids a search.
3499+ */
3500+ if (likely (!alloc_harder ))
3501+ unusable_free += z -> nr_reserved_highatomic ;
3502+
3503+ #ifdef CONFIG_CMA
3504+ /* If allocation can't use CMA areas don't use free CMA pages */
3505+ if (!(alloc_flags & ALLOC_CMA ))
3506+ unusable_free += zone_page_state (z , NR_FREE_CMA_PAGES );
3507+ #endif
3508+
3509+ return unusable_free ;
3510+ }
3511+
34893512/*
34903513 * Return true if free base pages are above 'mark'. For high-order checks it
34913514 * will return true of the order-0 watermark is reached and there is at least
@@ -3501,19 +3524,12 @@ bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
35013524 const bool alloc_harder = (alloc_flags & (ALLOC_HARDER |ALLOC_OOM ));
35023525
35033526 /* free_pages may go negative - that's OK */
3504- free_pages -= ( 1 << order ) - 1 ;
3527+ free_pages -= __zone_watermark_unusable_free ( z , order , alloc_flags ) ;
35053528
35063529 if (alloc_flags & ALLOC_HIGH )
35073530 min -= min / 2 ;
35083531
3509- /*
3510- * If the caller does not have rights to ALLOC_HARDER then subtract
3511- * the high-atomic reserves. This will over-estimate the size of the
3512- * atomic reserve but it avoids a search.
3513- */
3514- if (likely (!alloc_harder )) {
3515- free_pages -= z -> nr_reserved_highatomic ;
3516- } else {
3532+ if (unlikely (alloc_harder )) {
35173533 /*
35183534 * OOM victims can try even harder than normal ALLOC_HARDER
35193535 * users on the grounds that it's definitely going to be in
@@ -3526,13 +3542,6 @@ bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
35263542 min -= min / 4 ;
35273543 }
35283544
3529-
3530- #ifdef CONFIG_CMA
3531- /* If allocation can't use CMA areas don't use free CMA pages */
3532- if (!(alloc_flags & ALLOC_CMA ))
3533- free_pages -= zone_page_state (z , NR_FREE_CMA_PAGES );
3534- #endif
3535-
35363545 /*
35373546 * Check watermarks for an order-0 allocation request. If these
35383547 * are not met, then a high-order request also cannot go ahead
@@ -3581,25 +3590,22 @@ static inline bool zone_watermark_fast(struct zone *z, unsigned int order,
35813590 unsigned long mark , int highest_zoneidx ,
35823591 unsigned int alloc_flags )
35833592{
3584- long free_pages = zone_page_state (z , NR_FREE_PAGES );
3585- long cma_pages = 0 ;
3593+ long free_pages ;
35863594
3587- #ifdef CONFIG_CMA
3588- /* If allocation can't use CMA areas don't use free CMA pages */
3589- if (!(alloc_flags & ALLOC_CMA ))
3590- cma_pages = zone_page_state (z , NR_FREE_CMA_PAGES );
3591- #endif
3595+ free_pages = zone_page_state (z , NR_FREE_PAGES );
35923596
35933597 /*
35943598 * Fast check for order-0 only. If this fails then the reserves
3595- * need to be calculated. There is a corner case where the check
3596- * passes but only the high-order atomic reserve are free. If
3597- * the caller is !atomic then it'll uselessly search the free
3598- * list. That corner case is then slower but it is harmless.
3599+ * need to be calculated.
35993600 */
3600- if (!order && (free_pages - cma_pages ) >
3601- mark + z -> lowmem_reserve [highest_zoneidx ])
3602- return true;
3601+ if (!order ) {
3602+ long fast_free ;
3603+
3604+ fast_free = free_pages ;
3605+ fast_free -= __zone_watermark_unusable_free (z , 0 , alloc_flags );
3606+ if (fast_free > mark + z -> lowmem_reserve [highest_zoneidx ])
3607+ return true;
3608+ }
36033609
36043610 return __zone_watermark_ok (z , order , mark , highest_zoneidx , alloc_flags ,
36053611 free_pages );
0 commit comments