diff --git a/mm/mm_init.c b/mm/mm_init.c index 57b256ea9e6c..7e5e6984bb5d 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -2046,114 +2046,51 @@ static unsigned long __init deferred_init_pages(struct zone *zone, } /* - * This function is meant to pre-load the iterator for the zone init from - * a given point. - * Specifically it walks through the ranges starting with initial index - * passed to it until we are caught up to the first_init_pfn value and - * exits there. If we never encounter the value we return false indicating - * there are no valid ranges left. - */ -static bool __init -deferred_init_mem_pfn_range_in_zone(u64 *i, struct zone *zone, - unsigned long *spfn, unsigned long *epfn, - unsigned long first_init_pfn) -{ - u64 j = *i; - - if (j == 0) - __next_mem_pfn_range_in_zone(&j, zone, spfn, epfn); - - /* - * Start out by walking through the ranges in this zone that have - * already been initialized. We don't need to do anything with them - * so we just need to flush them out of the system. - */ - for_each_free_mem_pfn_range_in_zone_from(j, zone, spfn, epfn) { - if (*epfn <= first_init_pfn) - continue; - if (*spfn < first_init_pfn) - *spfn = first_init_pfn; - *i = j; - return true; - } - - return false; -} - -/* - * Initialize and free pages. We do it in two loops: first we initialize - * struct page, then free to buddy allocator, because while we are - * freeing pages we can access pages that are ahead (computing buddy - * page in __free_one_page()). + * Initialize and free pages. * - * In order to try and keep some memory in the cache we have the loop - * broken along max page order boundaries. This way we will not cause - * any issues with the buddy page computation. + * At this point reserved pages and struct pages that correspond to holes in + * memblock.memory are already intialized so every free range has a valid + * memory map around it. + * This ensures that access of pages that are ahead of the range being + * initialized (computing buddy page in __free_one_page()) always reads a valid + * struct page. + * + * In order to try and improve CPU cache locality we have the loop broken along + * max page order boundaries. */ -static unsigned long __init -deferred_init_maxorder(u64 *i, struct zone *zone, unsigned long *start_pfn, - unsigned long *end_pfn) -{ - unsigned long mo_pfn = ALIGN(*start_pfn + 1, MAX_ORDER_NR_PAGES); - unsigned long spfn = *start_pfn, epfn = *end_pfn; - unsigned long nr_pages = 0; - u64 j = *i; - - /* First we loop through and initialize the page values */ - for_each_free_mem_pfn_range_in_zone_from(j, zone, start_pfn, end_pfn) { - unsigned long t; - - if (mo_pfn <= *start_pfn) - break; - - t = min(mo_pfn, *end_pfn); - nr_pages += deferred_init_pages(zone, *start_pfn, t); - - if (mo_pfn < *end_pfn) { - *start_pfn = mo_pfn; - break; - } - } - - /* Reset values and now loop through freeing pages as needed */ - swap(j, *i); - - for_each_free_mem_pfn_range_in_zone_from(j, zone, &spfn, &epfn) { - unsigned long t; - - if (mo_pfn <= spfn) - break; - - t = min(mo_pfn, epfn); - deferred_free_pages(spfn, t - spfn); - - if (mo_pfn <= epfn) - break; - } - - return nr_pages; -} - static unsigned long __init deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn, struct zone *zone) { + int nid = zone_to_nid(zone); unsigned long nr_pages = 0; - unsigned long spfn, epfn; + phys_addr_t start, end; u64 i = 0; - deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, start_pfn); + for_each_free_mem_range(i, nid, 0, &start, &end, NULL) { + unsigned long spfn = PFN_UP(start); + unsigned long epfn = PFN_DOWN(end); - /* - * Initialize and free pages in MAX_PAGE_ORDER sized increments so that - * we can avoid introducing any issues with the buddy allocator. - */ - while (spfn < end_pfn) { - nr_pages += deferred_init_maxorder(&i, zone, &spfn, &epfn); - if (irqs_disabled()) - touch_nmi_watchdog(); - else - cond_resched(); + if (spfn >= end_pfn) + break; + + spfn = max(spfn, start_pfn); + epfn = min(epfn, end_pfn); + + while (spfn < epfn) { + unsigned long mo_pfn = ALIGN(spfn + 1, MAX_ORDER_NR_PAGES); + unsigned long chunk_end = min(mo_pfn, epfn); + + nr_pages += deferred_init_pages(zone, spfn, chunk_end); + deferred_free_pages(spfn, chunk_end - spfn); + + spfn = chunk_end; + + if (irqs_disabled()) + touch_nmi_watchdog(); + else + cond_resched(); + } } return nr_pages;