Skip to content

Commit 76741e7

Browse files
tehcastertorvalds
authored andcommitted
mm, page_alloc: don't convert pfn to idx when merging
In __free_one_page() we do the buddy merging arithmetics on "page/buddy index", which is just the lower MAX_ORDER bits of pfn. The operations we do that affect the higher bits are bitwise AND and subtraction (in that order), where the final result will be the same with the higher bits left unmasked, as long as these bits are equal for both buddies - which must be true by the definition of a buddy. We can therefore use pfn's directly instead of "index" and skip the zeroing of >MAX_ORDER bits. This can help a bit by itself, although compiler might be smart enough already. It also helps the next patch to avoid page_to_pfn() for memory hole checks. Link: http://lkml.kernel.org/r/[email protected] Signed-off-by: Vlastimil Babka <[email protected]> Acked-by: Mel Gorman <[email protected]> Cc: Joonsoo Kim <[email protected]> Cc: Michal Hocko <[email protected]> Cc: "Kirill A. Shutemov" <[email protected]> Cc: Johannes Weiner <[email protected]> Signed-off-by: Andrew Morton <[email protected]> Signed-off-by: Linus Torvalds <[email protected]>
1 parent aa18750 commit 76741e7

File tree

3 files changed

+20
-23
lines changed

3 files changed

+20
-23
lines changed

mm/internal.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -133,9 +133,9 @@ struct alloc_context {
133133
* Assumption: *_mem_map is contiguous at least up to MAX_ORDER
134134
*/
135135
static inline unsigned long
136-
__find_buddy_index(unsigned long page_idx, unsigned int order)
136+
__find_buddy_pfn(unsigned long page_pfn, unsigned int order)
137137
{
138-
return page_idx ^ (1 << order);
138+
return page_pfn ^ (1 << order);
139139
}
140140

141141
extern struct page *__pageblock_pfn_to_page(unsigned long start_pfn,

mm/page_alloc.c

Lines changed: 14 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -787,9 +787,8 @@ static inline void __free_one_page(struct page *page,
787787
struct zone *zone, unsigned int order,
788788
int migratetype)
789789
{
790-
unsigned long page_idx;
791-
unsigned long combined_idx;
792-
unsigned long uninitialized_var(buddy_idx);
790+
unsigned long combined_pfn;
791+
unsigned long uninitialized_var(buddy_pfn);
793792
struct page *buddy;
794793
unsigned int max_order;
795794

@@ -802,15 +801,13 @@ static inline void __free_one_page(struct page *page,
802801
if (likely(!is_migrate_isolate(migratetype)))
803802
__mod_zone_freepage_state(zone, 1 << order, migratetype);
804803

805-
page_idx = pfn & ((1 << MAX_ORDER) - 1);
806-
807-
VM_BUG_ON_PAGE(page_idx & ((1 << order) - 1), page);
804+
VM_BUG_ON_PAGE(pfn & ((1 << order) - 1), page);
808805
VM_BUG_ON_PAGE(bad_range(zone, page), page);
809806

810807
continue_merging:
811808
while (order < max_order - 1) {
812-
buddy_idx = __find_buddy_index(page_idx, order);
813-
buddy = page + (buddy_idx - page_idx);
809+
buddy_pfn = __find_buddy_pfn(pfn, order);
810+
buddy = page + (buddy_pfn - pfn);
814811
if (!page_is_buddy(page, buddy, order))
815812
goto done_merging;
816813
/*
@@ -824,9 +821,9 @@ static inline void __free_one_page(struct page *page,
824821
zone->free_area[order].nr_free--;
825822
rmv_page_order(buddy);
826823
}
827-
combined_idx = buddy_idx & page_idx;
828-
page = page + (combined_idx - page_idx);
829-
page_idx = combined_idx;
824+
combined_pfn = buddy_pfn & pfn;
825+
page = page + (combined_pfn - pfn);
826+
pfn = combined_pfn;
830827
order++;
831828
}
832829
if (max_order < MAX_ORDER) {
@@ -841,8 +838,8 @@ static inline void __free_one_page(struct page *page,
841838
if (unlikely(has_isolate_pageblock(zone))) {
842839
int buddy_mt;
843840

844-
buddy_idx = __find_buddy_index(page_idx, order);
845-
buddy = page + (buddy_idx - page_idx);
841+
buddy_pfn = __find_buddy_pfn(pfn, order);
842+
buddy = page + (buddy_pfn - pfn);
846843
buddy_mt = get_pageblock_migratetype(buddy);
847844

848845
if (migratetype != buddy_mt
@@ -867,10 +864,10 @@ static inline void __free_one_page(struct page *page,
867864
*/
868865
if ((order < MAX_ORDER-2) && pfn_valid_within(page_to_pfn(buddy))) {
869866
struct page *higher_page, *higher_buddy;
870-
combined_idx = buddy_idx & page_idx;
871-
higher_page = page + (combined_idx - page_idx);
872-
buddy_idx = __find_buddy_index(combined_idx, order + 1);
873-
higher_buddy = higher_page + (buddy_idx - combined_idx);
867+
combined_pfn = buddy_pfn & pfn;
868+
higher_page = page + (combined_pfn - pfn);
869+
buddy_pfn = __find_buddy_pfn(combined_pfn, order + 1);
870+
higher_buddy = higher_page + (buddy_pfn - combined_pfn);
874871
if (page_is_buddy(higher_page, higher_buddy, order + 1)) {
875872
list_add_tail(&page->lru,
876873
&zone->free_area[order].free_list[migratetype]);

mm/page_isolation.c

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ static void unset_migratetype_isolate(struct page *page, unsigned migratetype)
8383
unsigned long flags, nr_pages;
8484
bool isolated_page = false;
8585
unsigned int order;
86-
unsigned long page_idx, buddy_idx;
86+
unsigned long pfn, buddy_pfn;
8787
struct page *buddy;
8888

8989
zone = page_zone(page);
@@ -102,9 +102,9 @@ static void unset_migratetype_isolate(struct page *page, unsigned migratetype)
102102
if (PageBuddy(page)) {
103103
order = page_order(page);
104104
if (order >= pageblock_order) {
105-
page_idx = page_to_pfn(page) & ((1 << MAX_ORDER) - 1);
106-
buddy_idx = __find_buddy_index(page_idx, order);
107-
buddy = page + (buddy_idx - page_idx);
105+
pfn = page_to_pfn(page);
106+
buddy_pfn = __find_buddy_pfn(pfn, order);
107+
buddy = page + (buddy_pfn - pfn);
108108

109109
if (pfn_valid_within(page_to_pfn(buddy)) &&
110110
!is_migrate_isolate_page(buddy)) {

0 commit comments

Comments
 (0)