Merge branches 'bigrtm.2012.07.04a', 'doctorture.2012.07.02a', 'fixes.2012.07.06a...

[firefly-linux-kernel-4.4.55.git] / kernel / rcutree_plugin.h

diff --git a/kernel/rcutree_plugin.h b/kernel/rcutree_plugin.h
index d18b4d383afe11fc8f61ef8d948c491d49bb9732..a9194d5606c48e68ef67a64ec4ba4b2943f8881a 100644 (file)
--- a/kernel/rcutree_plugin.h
+++ b/kernel/rcutree_plugin.h
@@ -83,7 +83,6 @@ struct rcu_state rcu_preempt_state =
 DEFINE_PER_CPU(struct rcu_data, rcu_preempt_data);
 static struct rcu_state *rcu_state = &rcu_preempt_state;
 
-static void rcu_read_unlock_special(struct task_struct *t);
 static int rcu_preempted_readers_exp(struct rcu_node *rnp);
 
 /*
@@ -237,18 +236,6 @@ static void rcu_preempt_note_context_switch(int cpu)
        local_irq_restore(flags);
 }
 
-/*
- * Tree-preemptible RCU implementation for rcu_read_lock().
- * Just increment ->rcu_read_lock_nesting, shared state will be updated
- * if we block.
- */
-void __rcu_read_lock(void)
-{
-       current->rcu_read_lock_nesting++;
-       barrier();  /* needed if we ever invoke rcu_read_lock in rcutree.c */
-}
-EXPORT_SYMBOL_GPL(__rcu_read_lock);
-
 /*
  * Check for preempted RCU readers blocking the current grace period
  * for the specified rcu_node structure.  If the caller needs a reliable
@@ -315,7 +302,7 @@ static struct list_head *rcu_next_node_entry(struct task_struct *t,
  * notify RCU core processing or task having blocked during the RCU
  * read-side critical section.
  */
-static noinline void rcu_read_unlock_special(struct task_struct *t)
+void rcu_read_unlock_special(struct task_struct *t)
 {
        int empty;
        int empty_exp;
@@ -423,38 +410,6 @@ static noinline void rcu_read_unlock_special(struct task_struct *t)
        }
 }
 
-/*
- * Tree-preemptible RCU implementation for rcu_read_unlock().
- * Decrement ->rcu_read_lock_nesting.  If the result is zero (outermost
- * rcu_read_unlock()) and ->rcu_read_unlock_special is non-zero, then
- * invoke rcu_read_unlock_special() to clean up after a context switch
- * in an RCU read-side critical section and other special cases.
- */
-void __rcu_read_unlock(void)
-{
-       struct task_struct *t = current;
-
-       if (t->rcu_read_lock_nesting != 1)
-               --t->rcu_read_lock_nesting;
-       else {
-               barrier();  /* critical section before exit code. */
-               t->rcu_read_lock_nesting = INT_MIN;
-               barrier();  /* assign before ->rcu_read_unlock_special load */
-               if (unlikely(ACCESS_ONCE(t->rcu_read_unlock_special)))
-                       rcu_read_unlock_special(t);
-               barrier();  /* ->rcu_read_unlock_special load before assign */
-               t->rcu_read_lock_nesting = 0;
-       }
-#ifdef CONFIG_PROVE_LOCKING
-       {
-               int rrln = ACCESS_ONCE(t->rcu_read_lock_nesting);
-
-               WARN_ON_ONCE(rrln < 0 && rrln > INT_MIN / 2);
-       }
-#endif /* #ifdef CONFIG_PROVE_LOCKING */
-}
-EXPORT_SYMBOL_GPL(__rcu_read_unlock);
-
 #ifdef CONFIG_RCU_CPU_STALL_VERBOSE
 
 /*
@@ -1856,9 +1811,11 @@ static void rcu_idle_count_callbacks_posted(void)
  */
 #define RCU_IDLE_FLUSHES 5             /* Number of dyntick-idle tries. */
 #define RCU_IDLE_OPT_FLUSHES 3         /* Optional dyntick-idle tries. */
-#define RCU_IDLE_GP_DELAY 6            /* Roughly one grace period. */
+#define RCU_IDLE_GP_DELAY 4            /* Roughly one grace period. */
 #define RCU_IDLE_LAZY_GP_DELAY (6 * HZ)        /* Roughly six seconds. */
 
+extern int tick_nohz_enabled;
+
 /*
  * Does the specified flavor of RCU have non-lazy callbacks pending on
  * the specified CPU?  Both RCU flavor and CPU are specified by the
@@ -1935,10 +1892,13 @@ int rcu_needs_cpu(int cpu, unsigned long *delta_jiffies)
                return 1;
        }
        /* Set up for the possibility that RCU will post a timer. */
-       if (rcu_cpu_has_nonlazy_callbacks(cpu))
-               *delta_jiffies = RCU_IDLE_GP_DELAY;
-       else
-               *delta_jiffies = RCU_IDLE_LAZY_GP_DELAY;
+       if (rcu_cpu_has_nonlazy_callbacks(cpu)) {
+               *delta_jiffies = round_up(RCU_IDLE_GP_DELAY + jiffies,
+                                         RCU_IDLE_GP_DELAY) - jiffies;
+       } else {
+               *delta_jiffies = jiffies + RCU_IDLE_LAZY_GP_DELAY;
+               *delta_jiffies = round_jiffies(*delta_jiffies) - jiffies;
+       }
        return 0;
 }
 
@@ -1997,6 +1957,7 @@ static void rcu_cleanup_after_idle(int cpu)
 
        del_timer(&rdtp->idle_gp_timer);
        trace_rcu_prep_idle("Cleanup after idle");
+       rdtp->tick_nohz_enabled_snap = ACCESS_ONCE(tick_nohz_enabled);
 }
 
 /*
@@ -2022,6 +1983,18 @@ static void rcu_prepare_for_idle(int cpu)
 {
        struct timer_list *tp;
        struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
+       int tne;
+
+       /* Handle nohz enablement switches conservatively. */
+       tne = ACCESS_ONCE(tick_nohz_enabled);
+       if (tne != rdtp->tick_nohz_enabled_snap) {
+               if (rcu_cpu_has_callbacks(cpu))
+                       invoke_rcu_core(); /* force nohz to see update. */
+               rdtp->tick_nohz_enabled_snap = tne;
+               return;
+       }
+       if (!tne)
+               return;
 
        /*
         * If this is an idle re-entry, for example, due to use of
@@ -2075,10 +2048,11 @@ static void rcu_prepare_for_idle(int cpu)
                if (rcu_cpu_has_nonlazy_callbacks(cpu)) {
                        trace_rcu_prep_idle("Dyntick with callbacks");
                        rdtp->idle_gp_timer_expires =
-                                          jiffies + RCU_IDLE_GP_DELAY;
+                               round_up(jiffies + RCU_IDLE_GP_DELAY,
+                                        RCU_IDLE_GP_DELAY);
                } else {
                        rdtp->idle_gp_timer_expires =
-                                          jiffies + RCU_IDLE_LAZY_GP_DELAY;
+                               round_jiffies(jiffies + RCU_IDLE_LAZY_GP_DELAY);
                        trace_rcu_prep_idle("Dyntick with lazy callbacks");
                }
                tp = &rdtp->idle_gp_timer;
@@ -2157,6 +2131,7 @@ static void print_cpu_stall_fast_no_hz(char *cp, int cpu)
 
 static void print_cpu_stall_fast_no_hz(char *cp, int cpu)
 {
+       *cp = '\0';
 }
 
 #endif /* #else #ifdef CONFIG_RCU_FAST_NO_HZ */