Merge git://git.kernel.org/pub/scm/linux/kernel/git/davem/sparc-2.6

[firefly-linux-kernel-4.4.55.git] / mm / page-writeback.c

diff --git a/mm/page-writeback.c b/mm/page-writeback.c
index 2cf69a5e46e624a02767d564be4ce4a0daacf842..c09ef5219cbe36f267a37f55d6fc670815082522 100644 (file)
--- a/mm/page-writeback.c
+++ b/mm/page-writeback.c
@@ -261,16 +261,24 @@ static inline void task_dirties_fraction(struct task_struct *tsk,
 }
 
 /*
- * scale the dirty limit
+ * task_dirty_limit - scale down dirty throttling threshold for one task
  *
  * task specific dirty limit:
  *
  *   dirty -= (dirty/8) * p_{t}
+ *
+ * To protect light/slow dirtying tasks from heavier/fast ones, we start
+ * throttling individual tasks before reaching the bdi dirty limit.
+ * Relatively low thresholds will be allocated to heavy dirtiers. So when
+ * dirty pages grow large, heavy dirtiers will be throttled first, which will
+ * effectively curb the growth of dirty pages. Light dirtiers with high enough
+ * dirty threshold may never get throttled.
  */
-static void task_dirty_limit(struct task_struct *tsk, unsigned long *pdirty)
+static unsigned long task_dirty_limit(struct task_struct *tsk,
+                                      unsigned long bdi_dirty)
 {
        long numerator, denominator;
-       unsigned long dirty = *pdirty;
+       unsigned long dirty = bdi_dirty;
        u64 inv = dirty >> 3;
 
        task_dirties_fraction(tsk, &numerator, &denominator);
@@ -278,10 +286,8 @@ static void task_dirty_limit(struct task_struct *tsk, unsigned long *pdirty)
        do_div(inv, denominator);
 
        dirty -= inv;
-       if (dirty < *pdirty/2)
-               dirty = *pdirty/2;
 
-       *pdirty = dirty;
+       return max(dirty, bdi_dirty/2);
 }
 
 /*
@@ -391,9 +397,16 @@ unsigned long determine_dirtyable_memory(void)
        return x + 1;   /* Ensure that we never return 0 */
 }
 
-void
-get_dirty_limits(unsigned long *pbackground, unsigned long *pdirty,
-                unsigned long *pbdi_dirty, struct backing_dev_info *bdi)
+/*
+ * global_dirty_limits - background-writeback and dirty-throttling thresholds
+ *
+ * Calculate the dirty thresholds based on sysctl parameters
+ * - vm.dirty_background_ratio  or  vm.dirty_background_bytes
+ * - vm.dirty_ratio             or  vm.dirty_bytes
+ * The dirty limits will be lifted by 1/4 for PF_LESS_THROTTLE (ie. nfsd) and
+ * runtime tasks.
+ */
+void global_dirty_limits(unsigned long *pbackground, unsigned long *pdirty)
 {
        unsigned long background;
        unsigned long dirty;
@@ -425,26 +438,37 @@ get_dirty_limits(unsigned long *pbackground, unsigned long *pdirty,
        }
        *pbackground = background;
        *pdirty = dirty;
+}
+
+/*
+ * bdi_dirty_limit - @bdi's share of dirty throttling threshold
+ *
+ * Allocate high/low dirty limits to fast/slow devices, in order to prevent
+ * - starving fast devices
+ * - piling up dirty pages (that will take long time to sync) on slow devices
+ *
+ * The bdi's share of dirty limit will be adapting to its throughput and
+ * bounded by the bdi->min_ratio and/or bdi->max_ratio parameters, if set.
+ */
+unsigned long bdi_dirty_limit(struct backing_dev_info *bdi, unsigned long dirty)
+{
+       u64 bdi_dirty;
+       long numerator, denominator;
 
-       if (bdi) {
-               u64 bdi_dirty;
-               long numerator, denominator;
+       /*
+        * Calculate this BDI's share of the dirty ratio.
+        */
+       bdi_writeout_fraction(bdi, &numerator, &denominator);
 
-               /*
-                * Calculate this BDI's share of the dirty ratio.
-                */
-               bdi_writeout_fraction(bdi, &numerator, &denominator);
+       bdi_dirty = (dirty * (100 - bdi_min_ratio)) / 100;
+       bdi_dirty *= numerator;
+       do_div(bdi_dirty, denominator);
 
-               bdi_dirty = (dirty * (100 - bdi_min_ratio)) / 100;
-               bdi_dirty *= numerator;
-               do_div(bdi_dirty, denominator);
-               bdi_dirty += (dirty * bdi->min_ratio) / 100;
-               if (bdi_dirty > (dirty * bdi->max_ratio) / 100)
-                       bdi_dirty = dirty * bdi->max_ratio / 100;
+       bdi_dirty += (dirty * bdi->min_ratio) / 100;
+       if (bdi_dirty > (dirty * bdi->max_ratio) / 100)
+               bdi_dirty = dirty * bdi->max_ratio / 100;
 
-               *pbdi_dirty = bdi_dirty;
-               task_dirty_limit(current, pbdi_dirty);
-       }
+       return bdi_dirty;
 }
 
 /*
@@ -475,13 +499,24 @@ static void balance_dirty_pages(struct address_space *mapping,
                        .range_cyclic   = 1,
                };
 
-               get_dirty_limits(&background_thresh, &dirty_thresh,
-                               &bdi_thresh, bdi);
-
                nr_reclaimable = global_page_state(NR_FILE_DIRTY) +
                                        global_page_state(NR_UNSTABLE_NFS);
                nr_writeback = global_page_state(NR_WRITEBACK);
 
+               global_dirty_limits(&background_thresh, &dirty_thresh);
+
+               /*
+                * Throttle it only when the background writeback cannot
+                * catch-up. This avoids (excessively) small writeouts
+                * when the bdi limits are ramping up.
+                */
+               if (nr_reclaimable + nr_writeback <
+                               (background_thresh + dirty_thresh) / 2)
+                       break;
+
+               bdi_thresh = bdi_dirty_limit(bdi, dirty_thresh);
+               bdi_thresh = task_dirty_limit(current, bdi_thresh);
+
                /*
                 * In order to avoid the stacked BDI deadlock we need
                 * to ensure we accurately count the 'dirty' pages when
@@ -513,15 +548,6 @@ static void balance_dirty_pages(struct address_space *mapping,
                if (!dirty_exceeded)
                        break;
 
-               /*
-                * Throttle it only when the background writeback cannot
-                * catch-up. This avoids (excessively) small writeouts
-                * when the bdi limits are ramping up.
-                */
-               if (nr_reclaimable + nr_writeback <
-                               (background_thresh + dirty_thresh) / 2)
-                       break;
-
                if (!bdi->dirty_exceeded)
                        bdi->dirty_exceeded = 1;
 
@@ -634,7 +660,7 @@ void throttle_vm_writeout(gfp_t gfp_mask)
        unsigned long dirty_thresh;
 
         for ( ; ; ) {
-               get_dirty_limits(&background_thresh, &dirty_thresh, NULL, NULL);
+               global_dirty_limits(&background_thresh, &dirty_thresh);
 
                 /*
                  * Boost the allowable dirty threshold a bit for page
@@ -810,7 +836,8 @@ void tag_pages_for_writeback(struct address_space *mapping,
                spin_unlock_irq(&mapping->tree_lock);
                WARN_ON_ONCE(tagged > WRITEBACK_TAG_BATCH);
                cond_resched();
-       } while (tagged >= WRITEBACK_TAG_BATCH);
+               /* We check 'start' to handle wrapping when end == ~0UL */
+       } while (tagged >= WRITEBACK_TAG_BATCH && start);
 }
 EXPORT_SYMBOL(tag_pages_for_writeback);