Merge tag 'irqdomain-for-linus' of git://git.secretlab.ca/git/linux-2.6

[firefly-linux-kernel-4.4.55.git] / mm / page_alloc.c

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 228194728ccd1d7ba7c89f1c5665ff8b6d460389..889532b8e6c10342d7355c8868cf5812ec8de500 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -218,6 +218,11 @@ EXPORT_SYMBOL(nr_online_nodes);
 
 int page_group_by_mobility_disabled __read_mostly;
 
+/*
+ * NOTE:
+ * Don't use set_pageblock_migratetype(page, MIGRATE_ISOLATE) directly.
+ * Instead, use {un}set_pageblock_isolate.
+ */
 void set_pageblock_migratetype(struct page *page, int migratetype)
 {
 
@@ -1595,6 +1600,7 @@ static bool __zone_watermark_ok(struct zone *z, int order, unsigned long mark,
 {
        /* free_pages my go negative - that's OK */
        long min = mark;
+       long lowmem_reserve = z->lowmem_reserve[classzone_idx];
        int o;
 
        free_pages -= (1 << order) - 1;
@@ -1603,7 +1609,7 @@ static bool __zone_watermark_ok(struct zone *z, int order, unsigned long mark,
        if (alloc_flags & ALLOC_HARDER)
                min -= min / 4;
 
-       if (free_pages <= min + z->lowmem_reserve[classzone_idx])
+       if (free_pages <= min + lowmem_reserve)
                return false;
        for (o = 0; o < order; o++) {
                /* At the next order, this order's pages become unavailable */
@@ -1618,6 +1624,20 @@ static bool __zone_watermark_ok(struct zone *z, int order, unsigned long mark,
        return true;
 }
 
+#ifdef CONFIG_MEMORY_ISOLATION
+static inline unsigned long nr_zone_isolate_freepages(struct zone *zone)
+{
+       if (unlikely(zone->nr_pageblock_isolate))
+               return zone->nr_pageblock_isolate * pageblock_nr_pages;
+       return 0;
+}
+#else
+static inline unsigned long nr_zone_isolate_freepages(struct zone *zone)
+{
+       return 0;
+}
+#endif
+
 bool zone_watermark_ok(struct zone *z, int order, unsigned long mark,
                      int classzone_idx, int alloc_flags)
 {
@@ -1633,6 +1653,14 @@ bool zone_watermark_ok_safe(struct zone *z, int order, unsigned long mark,
        if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark)
                free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES);
 
+       /*
+        * If the zone has MIGRATE_ISOLATE type free pages, we should consider
+        * it.  nr_zone_isolate_freepages is never accurate so kswapd might not
+        * sleep although it could do so.  But this is more desirable for memory
+        * hotplug than sleeping which can cause a livelock in the direct
+        * reclaim path.
+        */
+       free_pages -= nr_zone_isolate_freepages(z);
        return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags,
                                                                free_pages);
 }
@@ -2088,8 +2116,8 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
 
                page = get_page_from_freelist(gfp_mask, nodemask,
                                order, zonelist, high_zoneidx,
-                               alloc_flags, preferred_zone,
-                               migratetype);
+                               alloc_flags & ~ALLOC_NO_WATERMARKS,
+                               preferred_zone, migratetype);
                if (page) {
                        preferred_zone->compact_considered = 0;
                        preferred_zone->compact_defer_shift = 0;
@@ -2181,8 +2209,8 @@ __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order,
 retry:
        page = get_page_from_freelist(gfp_mask, nodemask, order,
                                        zonelist, high_zoneidx,
-                                       alloc_flags, preferred_zone,
-                                       migratetype);
+                                       alloc_flags & ~ALLOC_NO_WATERMARKS,
+                                       preferred_zone, migratetype);
 
        /*
         * If an allocation failed after direct reclaim, it could be because
@@ -2266,15 +2294,24 @@ gfp_to_alloc_flags(gfp_t gfp_mask)
                alloc_flags |= ALLOC_HARDER;
 
        if (likely(!(gfp_mask & __GFP_NOMEMALLOC))) {
-               if (!in_interrupt() &&
-                   ((current->flags & PF_MEMALLOC) ||
-                    unlikely(test_thread_flag(TIF_MEMDIE))))
+               if (gfp_mask & __GFP_MEMALLOC)
+                       alloc_flags |= ALLOC_NO_WATERMARKS;
+               else if (in_serving_softirq() && (current->flags & PF_MEMALLOC))
+                       alloc_flags |= ALLOC_NO_WATERMARKS;
+               else if (!in_interrupt() &&
+                               ((current->flags & PF_MEMALLOC) ||
+                                unlikely(test_thread_flag(TIF_MEMDIE))))
                        alloc_flags |= ALLOC_NO_WATERMARKS;
        }
 
        return alloc_flags;
 }
 
+bool gfp_pfmemalloc_allowed(gfp_t gfp_mask)
+{
+       return !!(gfp_to_alloc_flags(gfp_mask) & ALLOC_NO_WATERMARKS);
+}
+
 static inline struct page *
 __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
        struct zonelist *zonelist, enum zone_type high_zoneidx,
@@ -2341,11 +2378,27 @@ rebalance:
 
        /* Allocate without watermarks if the context allows */
        if (alloc_flags & ALLOC_NO_WATERMARKS) {
+               /*
+                * Ignore mempolicies if ALLOC_NO_WATERMARKS on the grounds
+                * the allocation is high priority and these type of
+                * allocations are system rather than user orientated
+                */
+               zonelist = node_zonelist(numa_node_id(), gfp_mask);
+
                page = __alloc_pages_high_priority(gfp_mask, order,
                                zonelist, high_zoneidx, nodemask,
                                preferred_zone, migratetype);
-               if (page)
+               if (page) {
+                       /*
+                        * page->pfmemalloc is set when ALLOC_NO_WATERMARKS was
+                        * necessary to allocate the page. The expectation is
+                        * that the caller is taking steps that will free more
+                        * memory. The caller should avoid the page being used
+                        * for !PFMEMALLOC purposes.
+                        */
+                       page->pfmemalloc = true;
                        goto got_pg;
+               }
        }
 
        /* Atomic allocations - we can't balance anything */
@@ -2464,8 +2517,8 @@ nopage:
 got_pg:
        if (kmemcheck_enabled)
                kmemcheck_pagealloc_alloc(page, order, gfp_mask);
-       return page;
 
+       return page;
 }
 
 /*
@@ -2516,6 +2569,8 @@ retry_cpuset:
                page = __alloc_pages_slowpath(gfp_mask, order,
                                zonelist, high_zoneidx, nodemask,
                                preferred_zone, migratetype);
+       else
+               page->pfmemalloc = false;
 
        trace_mm_page_alloc(page, order, gfp_mask, migratetype);
 
@@ -4322,6 +4377,8 @@ void __init set_pageblock_order(void)
  *   - mark all pages reserved
  *   - mark all memory queues empty
  *   - clear the memory bitmaps
+ *
+ * NOTE: pgdat should get zeroed by caller.
  */
 static void __paginginit free_area_init_core(struct pglist_data *pgdat,
                unsigned long *zones_size, unsigned long *zholes_size)
@@ -4332,9 +4389,8 @@ static void __paginginit free_area_init_core(struct pglist_data *pgdat,
        int ret;
 
        pgdat_resize_init(pgdat);
-       pgdat->nr_zones = 0;
        init_waitqueue_head(&pgdat->kswapd_wait);
-       pgdat->kswapd_max_order = 0;
+       init_waitqueue_head(&pgdat->pfmemalloc_wait);
        pgdat_page_cgroup_init(pgdat);
 
        for (j = 0; j < MAX_NR_ZONES; j++) {
@@ -4395,8 +4451,6 @@ static void __paginginit free_area_init_core(struct pglist_data *pgdat,
 
                zone_pcp_init(zone);
                lruvec_init(&zone->lruvec, zone);
-               zap_zone_vm_stats(zone);
-               zone->flags = 0;
                if (!size)
                        continue;
 
@@ -4456,6 +4510,9 @@ void __paginginit free_area_init_node(int nid, unsigned long *zones_size,
 {
        pg_data_t *pgdat = NODE_DATA(nid);
 
+       /* pg_data_t should be reset to zero when it's allocated */
+       WARN_ON(pgdat->nr_zones || pgdat->node_start_pfn || pgdat->classzone_idx);
+
        pgdat->node_id = nid;
        pgdat->node_start_pfn = node_start_pfn;
        calculate_node_totalpages(pgdat, zones_size, zholes_size);