2 * Read-Copy Update mechanism for mutual exclusion (tree-based version)
3 * Internal non-public definitions that provide either classic
4 * or preemptible semantics.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
20 * Copyright Red Hat, 2009
21 * Copyright IBM Corporation, 2009
23 * Author: Ingo Molnar <mingo@elte.hu>
24 * Paul E. McKenney <paulmck@linux.vnet.ibm.com>
27 #include <linux/delay.h>
28 #include <linux/stop_machine.h>
31 * Check the RCU kernel configuration parameters and print informative
32 * messages about anything out of the ordinary. If you like #ifdef, you
33 * will love this function.
35 static void __init rcu_bootup_announce_oddness(void)
37 #ifdef CONFIG_RCU_TRACE
38 printk(KERN_INFO "\tRCU debugfs-based tracing is enabled.\n");
40 #if (defined(CONFIG_64BIT) && CONFIG_RCU_FANOUT != 64) || (!defined(CONFIG_64BIT) && CONFIG_RCU_FANOUT != 32)
41 printk(KERN_INFO "\tCONFIG_RCU_FANOUT set to non-default value of %d\n",
44 #ifdef CONFIG_RCU_FANOUT_EXACT
45 printk(KERN_INFO "\tHierarchical RCU autobalancing is disabled.\n");
47 #ifdef CONFIG_RCU_FAST_NO_HZ
49 "\tRCU dyntick-idle grace-period acceleration is enabled.\n");
51 #ifdef CONFIG_PROVE_RCU
52 printk(KERN_INFO "\tRCU lockdep checking is enabled.\n");
54 #ifdef CONFIG_RCU_TORTURE_TEST_RUNNABLE
55 printk(KERN_INFO "\tRCU torture testing starts during boot.\n");
57 #if defined(CONFIG_TREE_PREEMPT_RCU) && !defined(CONFIG_RCU_CPU_STALL_VERBOSE)
58 printk(KERN_INFO "\tVerbose stalled-CPUs detection is disabled.\n");
60 #if NUM_RCU_LVL_4 != 0
61 printk(KERN_INFO "\tExperimental four-level hierarchy is enabled.\n");
65 #ifdef CONFIG_TREE_PREEMPT_RCU
67 struct rcu_state rcu_preempt_state = RCU_STATE_INITIALIZER(rcu_preempt_state);
68 DEFINE_PER_CPU(struct rcu_data, rcu_preempt_data);
69 static struct rcu_state *rcu_state = &rcu_preempt_state;
71 static int rcu_preempted_readers_exp(struct rcu_node *rnp);
74 * Tell them what RCU they are running.
76 static void __init rcu_bootup_announce(void)
78 printk(KERN_INFO "Preemptible hierarchical RCU implementation.\n");
79 rcu_bootup_announce_oddness();
83 * Return the number of RCU-preempt batches processed thus far
84 * for debug and statistics.
86 long rcu_batches_completed_preempt(void)
88 return rcu_preempt_state.completed;
90 EXPORT_SYMBOL_GPL(rcu_batches_completed_preempt);
93 * Return the number of RCU batches processed thus far for debug & stats.
95 long rcu_batches_completed(void)
97 return rcu_batches_completed_preempt();
99 EXPORT_SYMBOL_GPL(rcu_batches_completed);
102 * Force a quiescent state for preemptible RCU.
104 void rcu_force_quiescent_state(void)
106 force_quiescent_state(&rcu_preempt_state, 0);
108 EXPORT_SYMBOL_GPL(rcu_force_quiescent_state);
111 * Record a preemptible-RCU quiescent state for the specified CPU. Note
112 * that this just means that the task currently running on the CPU is
113 * not in a quiescent state. There might be any number of tasks blocked
114 * while in an RCU read-side critical section.
116 * Unlike the other rcu_*_qs() functions, callers to this function
117 * must disable irqs in order to protect the assignment to
118 * ->rcu_read_unlock_special.
120 static void rcu_preempt_qs(int cpu)
122 struct rcu_data *rdp = &per_cpu(rcu_preempt_data, cpu);
124 rdp->passed_quiesc_completed = rdp->gpnum - 1;
126 rdp->passed_quiesc = 1;
127 current->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_NEED_QS;
131 * We have entered the scheduler, and the current task might soon be
132 * context-switched away from. If this task is in an RCU read-side
133 * critical section, we will no longer be able to rely on the CPU to
134 * record that fact, so we enqueue the task on the blkd_tasks list.
135 * The task will dequeue itself when it exits the outermost enclosing
136 * RCU read-side critical section. Therefore, the current grace period
137 * cannot be permitted to complete until the blkd_tasks list entries
138 * predating the current grace period drain, in other words, until
139 * rnp->gp_tasks becomes NULL.
141 * Caller must disable preemption.
143 static void rcu_preempt_note_context_switch(int cpu)
145 struct task_struct *t = current;
147 struct rcu_data *rdp;
148 struct rcu_node *rnp;
150 if (t->rcu_read_lock_nesting &&
151 (t->rcu_read_unlock_special & RCU_READ_UNLOCK_BLOCKED) == 0) {
153 /* Possibly blocking in an RCU read-side critical section. */
154 rdp = per_cpu_ptr(rcu_preempt_state.rda, cpu);
156 raw_spin_lock_irqsave(&rnp->lock, flags);
157 t->rcu_read_unlock_special |= RCU_READ_UNLOCK_BLOCKED;
158 t->rcu_blocked_node = rnp;
161 * If this CPU has already checked in, then this task
162 * will hold up the next grace period rather than the
163 * current grace period. Queue the task accordingly.
164 * If the task is queued for the current grace period
165 * (i.e., this CPU has not yet passed through a quiescent
166 * state for the current grace period), then as long
167 * as that task remains queued, the current grace period
168 * cannot end. Note that there is some uncertainty as
169 * to exactly when the current grace period started.
170 * We take a conservative approach, which can result
171 * in unnecessarily waiting on tasks that started very
172 * slightly after the current grace period began. C'est
175 * But first, note that the current CPU must still be
178 WARN_ON_ONCE((rdp->grpmask & rnp->qsmaskinit) == 0);
179 WARN_ON_ONCE(!list_empty(&t->rcu_node_entry));
180 if ((rnp->qsmask & rdp->grpmask) && rnp->gp_tasks != NULL) {
181 list_add(&t->rcu_node_entry, rnp->gp_tasks->prev);
182 rnp->gp_tasks = &t->rcu_node_entry;
183 #ifdef CONFIG_RCU_BOOST
184 if (rnp->boost_tasks != NULL)
185 rnp->boost_tasks = rnp->gp_tasks;
186 #endif /* #ifdef CONFIG_RCU_BOOST */
188 list_add(&t->rcu_node_entry, &rnp->blkd_tasks);
189 if (rnp->qsmask & rdp->grpmask)
190 rnp->gp_tasks = &t->rcu_node_entry;
192 raw_spin_unlock_irqrestore(&rnp->lock, flags);
196 * Either we were not in an RCU read-side critical section to
197 * begin with, or we have now recorded that critical section
198 * globally. Either way, we can now note a quiescent state
199 * for this CPU. Again, if we were in an RCU read-side critical
200 * section, and if that critical section was blocking the current
201 * grace period, then the fact that the task has been enqueued
202 * means that we continue to block the current grace period.
204 local_irq_save(flags);
206 local_irq_restore(flags);
210 * Tree-preemptible RCU implementation for rcu_read_lock().
211 * Just increment ->rcu_read_lock_nesting, shared state will be updated
214 void __rcu_read_lock(void)
216 current->rcu_read_lock_nesting++;
217 barrier(); /* needed if we ever invoke rcu_read_lock in rcutree.c */
219 EXPORT_SYMBOL_GPL(__rcu_read_lock);
222 * Check for preempted RCU readers blocking the current grace period
223 * for the specified rcu_node structure. If the caller needs a reliable
224 * answer, it must hold the rcu_node's ->lock.
226 static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp)
228 return rnp->gp_tasks != NULL;
232 * Record a quiescent state for all tasks that were previously queued
233 * on the specified rcu_node structure and that were blocking the current
234 * RCU grace period. The caller must hold the specified rnp->lock with
235 * irqs disabled, and this lock is released upon return, but irqs remain
238 static void rcu_report_unblock_qs_rnp(struct rcu_node *rnp, unsigned long flags)
239 __releases(rnp->lock)
242 struct rcu_node *rnp_p;
244 if (rnp->qsmask != 0 || rcu_preempt_blocked_readers_cgp(rnp)) {
245 raw_spin_unlock_irqrestore(&rnp->lock, flags);
246 return; /* Still need more quiescent states! */
252 * Either there is only one rcu_node in the tree,
253 * or tasks were kicked up to root rcu_node due to
254 * CPUs going offline.
256 rcu_report_qs_rsp(&rcu_preempt_state, flags);
260 /* Report up the rest of the hierarchy. */
262 raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
263 raw_spin_lock(&rnp_p->lock); /* irqs already disabled. */
264 rcu_report_qs_rnp(mask, &rcu_preempt_state, rnp_p, flags);
268 * Advance a ->blkd_tasks-list pointer to the next entry, instead
269 * returning NULL if at the end of the list.
271 static struct list_head *rcu_next_node_entry(struct task_struct *t,
272 struct rcu_node *rnp)
274 struct list_head *np;
276 np = t->rcu_node_entry.next;
277 if (np == &rnp->blkd_tasks)
283 * Handle special cases during rcu_read_unlock(), such as needing to
284 * notify RCU core processing or task having blocked during the RCU
285 * read-side critical section.
287 static void rcu_read_unlock_special(struct task_struct *t)
292 struct list_head *np;
293 struct rcu_node *rnp;
296 /* NMI handlers cannot block and cannot safely manipulate state. */
300 local_irq_save(flags);
303 * If RCU core is waiting for this CPU to exit critical section,
304 * let it know that we have done so.
306 special = t->rcu_read_unlock_special;
307 if (special & RCU_READ_UNLOCK_NEED_QS) {
308 rcu_preempt_qs(smp_processor_id());
311 /* Hardware IRQ handlers cannot block. */
313 local_irq_restore(flags);
317 /* Clean up if blocked during RCU read-side critical section. */
318 if (special & RCU_READ_UNLOCK_BLOCKED) {
319 t->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_BLOCKED;
322 * Remove this task from the list it blocked on. The
323 * task can migrate while we acquire the lock, but at
324 * most one time. So at most two passes through loop.
327 rnp = t->rcu_blocked_node;
328 raw_spin_lock(&rnp->lock); /* irqs already disabled. */
329 if (rnp == t->rcu_blocked_node)
331 raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
333 empty = !rcu_preempt_blocked_readers_cgp(rnp);
334 empty_exp = !rcu_preempted_readers_exp(rnp);
335 smp_mb(); /* ensure expedited fastpath sees end of RCU c-s. */
336 np = rcu_next_node_entry(t, rnp);
337 list_del_init(&t->rcu_node_entry);
338 if (&t->rcu_node_entry == rnp->gp_tasks)
340 if (&t->rcu_node_entry == rnp->exp_tasks)
342 #ifdef CONFIG_RCU_BOOST
343 if (&t->rcu_node_entry == rnp->boost_tasks)
344 rnp->boost_tasks = np;
345 #endif /* #ifdef CONFIG_RCU_BOOST */
346 t->rcu_blocked_node = NULL;
349 * If this was the last task on the current list, and if
350 * we aren't waiting on any CPUs, report the quiescent state.
351 * Note that rcu_report_unblock_qs_rnp() releases rnp->lock.
354 raw_spin_unlock_irqrestore(&rnp->lock, flags);
356 rcu_report_unblock_qs_rnp(rnp, flags);
358 #ifdef CONFIG_RCU_BOOST
359 /* Unboost if we were boosted. */
360 if (special & RCU_READ_UNLOCK_BOOSTED) {
361 t->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_BOOSTED;
362 rt_mutex_unlock(t->rcu_boost_mutex);
363 t->rcu_boost_mutex = NULL;
365 #endif /* #ifdef CONFIG_RCU_BOOST */
368 * If this was the last task on the expedited lists,
369 * then we need to report up the rcu_node hierarchy.
371 if (!empty_exp && !rcu_preempted_readers_exp(rnp))
372 rcu_report_exp_rnp(&rcu_preempt_state, rnp);
374 local_irq_restore(flags);
379 * Tree-preemptible RCU implementation for rcu_read_unlock().
380 * Decrement ->rcu_read_lock_nesting. If the result is zero (outermost
381 * rcu_read_unlock()) and ->rcu_read_unlock_special is non-zero, then
382 * invoke rcu_read_unlock_special() to clean up after a context switch
383 * in an RCU read-side critical section and other special cases.
385 void __rcu_read_unlock(void)
387 struct task_struct *t = current;
389 barrier(); /* needed if we ever invoke rcu_read_unlock in rcutree.c */
390 --t->rcu_read_lock_nesting;
391 barrier(); /* decrement before load of ->rcu_read_unlock_special */
392 if (t->rcu_read_lock_nesting == 0 &&
393 unlikely(ACCESS_ONCE(t->rcu_read_unlock_special)))
394 rcu_read_unlock_special(t);
395 #ifdef CONFIG_PROVE_LOCKING
396 WARN_ON_ONCE(ACCESS_ONCE(t->rcu_read_lock_nesting) < 0);
397 #endif /* #ifdef CONFIG_PROVE_LOCKING */
399 EXPORT_SYMBOL_GPL(__rcu_read_unlock);
401 #ifdef CONFIG_RCU_CPU_STALL_VERBOSE
404 * Dump detailed information for all tasks blocking the current RCU
405 * grace period on the specified rcu_node structure.
407 static void rcu_print_detail_task_stall_rnp(struct rcu_node *rnp)
410 struct task_struct *t;
412 if (!rcu_preempt_blocked_readers_cgp(rnp))
414 raw_spin_lock_irqsave(&rnp->lock, flags);
415 t = list_entry(rnp->gp_tasks,
416 struct task_struct, rcu_node_entry);
417 list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry)
419 raw_spin_unlock_irqrestore(&rnp->lock, flags);
423 * Dump detailed information for all tasks blocking the current RCU
426 static void rcu_print_detail_task_stall(struct rcu_state *rsp)
428 struct rcu_node *rnp = rcu_get_root(rsp);
430 rcu_print_detail_task_stall_rnp(rnp);
431 rcu_for_each_leaf_node(rsp, rnp)
432 rcu_print_detail_task_stall_rnp(rnp);
435 #else /* #ifdef CONFIG_RCU_CPU_STALL_VERBOSE */
437 static void rcu_print_detail_task_stall(struct rcu_state *rsp)
441 #endif /* #else #ifdef CONFIG_RCU_CPU_STALL_VERBOSE */
444 * Scan the current list of tasks blocked within RCU read-side critical
445 * sections, printing out the tid of each.
447 static void rcu_print_task_stall(struct rcu_node *rnp)
449 struct task_struct *t;
451 if (!rcu_preempt_blocked_readers_cgp(rnp))
453 t = list_entry(rnp->gp_tasks,
454 struct task_struct, rcu_node_entry);
455 list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry)
456 printk(" P%d", t->pid);
460 * Suppress preemptible RCU's CPU stall warnings by pushing the
461 * time of the next stall-warning message comfortably far into the
464 static void rcu_preempt_stall_reset(void)
466 rcu_preempt_state.jiffies_stall = jiffies + ULONG_MAX / 2;
470 * Check that the list of blocked tasks for the newly completed grace
471 * period is in fact empty. It is a serious bug to complete a grace
472 * period that still has RCU readers blocked! This function must be
473 * invoked -before- updating this rnp's ->gpnum, and the rnp's ->lock
474 * must be held by the caller.
476 * Also, if there are blocked tasks on the list, they automatically
477 * block the newly created grace period, so set up ->gp_tasks accordingly.
479 static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp)
481 WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp));
482 if (!list_empty(&rnp->blkd_tasks))
483 rnp->gp_tasks = rnp->blkd_tasks.next;
484 WARN_ON_ONCE(rnp->qsmask);
487 #ifdef CONFIG_HOTPLUG_CPU
490 * Handle tasklist migration for case in which all CPUs covered by the
491 * specified rcu_node have gone offline. Move them up to the root
492 * rcu_node. The reason for not just moving them to the immediate
493 * parent is to remove the need for rcu_read_unlock_special() to
494 * make more than two attempts to acquire the target rcu_node's lock.
495 * Returns true if there were tasks blocking the current RCU grace
498 * Returns 1 if there was previously a task blocking the current grace
499 * period on the specified rcu_node structure.
501 * The caller must hold rnp->lock with irqs disabled.
503 static int rcu_preempt_offline_tasks(struct rcu_state *rsp,
504 struct rcu_node *rnp,
505 struct rcu_data *rdp)
507 struct list_head *lp;
508 struct list_head *lp_root;
510 struct rcu_node *rnp_root = rcu_get_root(rsp);
511 struct task_struct *t;
513 if (rnp == rnp_root) {
514 WARN_ONCE(1, "Last CPU thought to be offlined?");
515 return 0; /* Shouldn't happen: at least one CPU online. */
518 /* If we are on an internal node, complain bitterly. */
519 WARN_ON_ONCE(rnp != rdp->mynode);
522 * Move tasks up to root rcu_node. Don't try to get fancy for
523 * this corner-case operation -- just put this node's tasks
524 * at the head of the root node's list, and update the root node's
525 * ->gp_tasks and ->exp_tasks pointers to those of this node's,
526 * if non-NULL. This might result in waiting for more tasks than
527 * absolutely necessary, but this is a good performance/complexity
530 if (rcu_preempt_blocked_readers_cgp(rnp))
531 retval |= RCU_OFL_TASKS_NORM_GP;
532 if (rcu_preempted_readers_exp(rnp))
533 retval |= RCU_OFL_TASKS_EXP_GP;
534 lp = &rnp->blkd_tasks;
535 lp_root = &rnp_root->blkd_tasks;
536 while (!list_empty(lp)) {
537 t = list_entry(lp->next, typeof(*t), rcu_node_entry);
538 raw_spin_lock(&rnp_root->lock); /* irqs already disabled */
539 list_del(&t->rcu_node_entry);
540 t->rcu_blocked_node = rnp_root;
541 list_add(&t->rcu_node_entry, lp_root);
542 if (&t->rcu_node_entry == rnp->gp_tasks)
543 rnp_root->gp_tasks = rnp->gp_tasks;
544 if (&t->rcu_node_entry == rnp->exp_tasks)
545 rnp_root->exp_tasks = rnp->exp_tasks;
546 #ifdef CONFIG_RCU_BOOST
547 if (&t->rcu_node_entry == rnp->boost_tasks)
548 rnp_root->boost_tasks = rnp->boost_tasks;
549 #endif /* #ifdef CONFIG_RCU_BOOST */
550 raw_spin_unlock(&rnp_root->lock); /* irqs still disabled */
553 #ifdef CONFIG_RCU_BOOST
554 /* In case root is being boosted and leaf is not. */
555 raw_spin_lock(&rnp_root->lock); /* irqs already disabled */
556 if (rnp_root->boost_tasks != NULL &&
557 rnp_root->boost_tasks != rnp_root->gp_tasks)
558 rnp_root->boost_tasks = rnp_root->gp_tasks;
559 raw_spin_unlock(&rnp_root->lock); /* irqs still disabled */
560 #endif /* #ifdef CONFIG_RCU_BOOST */
562 rnp->gp_tasks = NULL;
563 rnp->exp_tasks = NULL;
568 * Do CPU-offline processing for preemptible RCU.
570 static void rcu_preempt_offline_cpu(int cpu)
572 __rcu_offline_cpu(cpu, &rcu_preempt_state);
575 #endif /* #ifdef CONFIG_HOTPLUG_CPU */
578 * Check for a quiescent state from the current CPU. When a task blocks,
579 * the task is recorded in the corresponding CPU's rcu_node structure,
580 * which is checked elsewhere.
582 * Caller must disable hard irqs.
584 static void rcu_preempt_check_callbacks(int cpu)
586 struct task_struct *t = current;
588 if (t->rcu_read_lock_nesting == 0) {
592 if (per_cpu(rcu_preempt_data, cpu).qs_pending)
593 t->rcu_read_unlock_special |= RCU_READ_UNLOCK_NEED_QS;
597 * Process callbacks for preemptible RCU.
599 static void rcu_preempt_process_callbacks(void)
601 __rcu_process_callbacks(&rcu_preempt_state,
602 &__get_cpu_var(rcu_preempt_data));
605 static void rcu_preempt_do_callbacks(void)
607 rcu_do_batch(&rcu_preempt_state, &__get_cpu_var(rcu_preempt_data));
611 * Queue a preemptible-RCU callback for invocation after a grace period.
613 void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
615 __call_rcu(head, func, &rcu_preempt_state);
617 EXPORT_SYMBOL_GPL(call_rcu);
620 * synchronize_rcu - wait until a grace period has elapsed.
622 * Control will return to the caller some time after a full grace
623 * period has elapsed, in other words after all currently executing RCU
624 * read-side critical sections have completed. Note, however, that
625 * upon return from synchronize_rcu(), the caller might well be executing
626 * concurrently with new RCU read-side critical sections that began while
627 * synchronize_rcu() was waiting. RCU read-side critical sections are
628 * delimited by rcu_read_lock() and rcu_read_unlock(), and may be nested.
630 void synchronize_rcu(void)
632 struct rcu_synchronize rcu;
634 if (!rcu_scheduler_active)
637 init_rcu_head_on_stack(&rcu.head);
638 init_completion(&rcu.completion);
639 /* Will wake me after RCU finished. */
640 call_rcu(&rcu.head, wakeme_after_rcu);
642 wait_for_completion(&rcu.completion);
643 destroy_rcu_head_on_stack(&rcu.head);
645 EXPORT_SYMBOL_GPL(synchronize_rcu);
647 static DECLARE_WAIT_QUEUE_HEAD(sync_rcu_preempt_exp_wq);
648 static long sync_rcu_preempt_exp_count;
649 static DEFINE_MUTEX(sync_rcu_preempt_exp_mutex);
652 * Return non-zero if there are any tasks in RCU read-side critical
653 * sections blocking the current preemptible-RCU expedited grace period.
654 * If there is no preemptible-RCU expedited grace period currently in
655 * progress, returns zero unconditionally.
657 static int rcu_preempted_readers_exp(struct rcu_node *rnp)
659 return rnp->exp_tasks != NULL;
663 * return non-zero if there is no RCU expedited grace period in progress
664 * for the specified rcu_node structure, in other words, if all CPUs and
665 * tasks covered by the specified rcu_node structure have done their bit
666 * for the current expedited grace period. Works only for preemptible
667 * RCU -- other RCU implementation use other means.
669 * Caller must hold sync_rcu_preempt_exp_mutex.
671 static int sync_rcu_preempt_exp_done(struct rcu_node *rnp)
673 return !rcu_preempted_readers_exp(rnp) &&
674 ACCESS_ONCE(rnp->expmask) == 0;
678 * Report the exit from RCU read-side critical section for the last task
679 * that queued itself during or before the current expedited preemptible-RCU
680 * grace period. This event is reported either to the rcu_node structure on
681 * which the task was queued or to one of that rcu_node structure's ancestors,
682 * recursively up the tree. (Calm down, calm down, we do the recursion
685 * Caller must hold sync_rcu_preempt_exp_mutex.
687 static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp)
692 raw_spin_lock_irqsave(&rnp->lock, flags);
694 if (!sync_rcu_preempt_exp_done(rnp))
696 if (rnp->parent == NULL) {
697 wake_up(&sync_rcu_preempt_exp_wq);
701 raw_spin_unlock(&rnp->lock); /* irqs remain disabled */
703 raw_spin_lock(&rnp->lock); /* irqs already disabled */
704 rnp->expmask &= ~mask;
706 raw_spin_unlock_irqrestore(&rnp->lock, flags);
710 * Snapshot the tasks blocking the newly started preemptible-RCU expedited
711 * grace period for the specified rcu_node structure. If there are no such
712 * tasks, report it up the rcu_node hierarchy.
714 * Caller must hold sync_rcu_preempt_exp_mutex and rsp->onofflock.
717 sync_rcu_preempt_exp_init(struct rcu_state *rsp, struct rcu_node *rnp)
722 raw_spin_lock_irqsave(&rnp->lock, flags);
723 if (list_empty(&rnp->blkd_tasks))
724 raw_spin_unlock_irqrestore(&rnp->lock, flags);
726 rnp->exp_tasks = rnp->blkd_tasks.next;
727 rcu_initiate_boost(rnp, flags); /* releases rnp->lock */
731 rcu_report_exp_rnp(rsp, rnp);
735 * Wait for an rcu-preempt grace period, but expedite it. The basic idea
736 * is to invoke synchronize_sched_expedited() to push all the tasks to
737 * the ->blkd_tasks lists and wait for this list to drain.
739 void synchronize_rcu_expedited(void)
742 struct rcu_node *rnp;
743 struct rcu_state *rsp = &rcu_preempt_state;
747 smp_mb(); /* Caller's modifications seen first by other CPUs. */
748 snap = ACCESS_ONCE(sync_rcu_preempt_exp_count) + 1;
749 smp_mb(); /* Above access cannot bleed into critical section. */
752 * Acquire lock, falling back to synchronize_rcu() if too many
753 * lock-acquisition failures. Of course, if someone does the
754 * expedited grace period for us, just leave.
756 while (!mutex_trylock(&sync_rcu_preempt_exp_mutex)) {
758 udelay(trycount * num_online_cpus());
763 if ((ACCESS_ONCE(sync_rcu_preempt_exp_count) - snap) > 0)
764 goto mb_ret; /* Others did our work for us. */
766 if ((ACCESS_ONCE(sync_rcu_preempt_exp_count) - snap) > 0)
767 goto unlock_mb_ret; /* Others did our work for us. */
769 /* force all RCU readers onto ->blkd_tasks lists. */
770 synchronize_sched_expedited();
772 raw_spin_lock_irqsave(&rsp->onofflock, flags);
774 /* Initialize ->expmask for all non-leaf rcu_node structures. */
775 rcu_for_each_nonleaf_node_breadth_first(rsp, rnp) {
776 raw_spin_lock(&rnp->lock); /* irqs already disabled. */
777 rnp->expmask = rnp->qsmaskinit;
778 raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
781 /* Snapshot current state of ->blkd_tasks lists. */
782 rcu_for_each_leaf_node(rsp, rnp)
783 sync_rcu_preempt_exp_init(rsp, rnp);
784 if (NUM_RCU_NODES > 1)
785 sync_rcu_preempt_exp_init(rsp, rcu_get_root(rsp));
787 raw_spin_unlock_irqrestore(&rsp->onofflock, flags);
789 /* Wait for snapshotted ->blkd_tasks lists to drain. */
790 rnp = rcu_get_root(rsp);
791 wait_event(sync_rcu_preempt_exp_wq,
792 sync_rcu_preempt_exp_done(rnp));
794 /* Clean up and exit. */
795 smp_mb(); /* ensure expedited GP seen before counter increment. */
796 ACCESS_ONCE(sync_rcu_preempt_exp_count)++;
798 mutex_unlock(&sync_rcu_preempt_exp_mutex);
800 smp_mb(); /* ensure subsequent action seen after grace period. */
802 EXPORT_SYMBOL_GPL(synchronize_rcu_expedited);
805 * Check to see if there is any immediate preemptible-RCU-related work
808 static int rcu_preempt_pending(int cpu)
810 return __rcu_pending(&rcu_preempt_state,
811 &per_cpu(rcu_preempt_data, cpu));
815 * Does preemptible RCU need the CPU to stay out of dynticks mode?
817 static int rcu_preempt_needs_cpu(int cpu)
819 return !!per_cpu(rcu_preempt_data, cpu).nxtlist;
823 * rcu_barrier - Wait until all in-flight call_rcu() callbacks complete.
825 void rcu_barrier(void)
827 _rcu_barrier(&rcu_preempt_state, call_rcu);
829 EXPORT_SYMBOL_GPL(rcu_barrier);
832 * Initialize preemptible RCU's per-CPU data.
834 static void __cpuinit rcu_preempt_init_percpu_data(int cpu)
836 rcu_init_percpu_data(cpu, &rcu_preempt_state, 1);
840 * Move preemptible RCU's callbacks from dying CPU to other online CPU.
842 static void rcu_preempt_send_cbs_to_online(void)
844 rcu_send_cbs_to_online(&rcu_preempt_state);
848 * Initialize preemptible RCU's state structures.
850 static void __init __rcu_init_preempt(void)
852 rcu_init_one(&rcu_preempt_state, &rcu_preempt_data);
856 * Check for a task exiting while in a preemptible-RCU read-side
857 * critical section, clean up if so. No need to issue warnings,
858 * as debug_check_no_locks_held() already does this if lockdep
863 struct task_struct *t = current;
865 if (t->rcu_read_lock_nesting == 0)
867 t->rcu_read_lock_nesting = 1;
871 #else /* #ifdef CONFIG_TREE_PREEMPT_RCU */
873 static struct rcu_state *rcu_state = &rcu_sched_state;
876 * Tell them what RCU they are running.
878 static void __init rcu_bootup_announce(void)
880 printk(KERN_INFO "Hierarchical RCU implementation.\n");
881 rcu_bootup_announce_oddness();
885 * Return the number of RCU batches processed thus far for debug & stats.
887 long rcu_batches_completed(void)
889 return rcu_batches_completed_sched();
891 EXPORT_SYMBOL_GPL(rcu_batches_completed);
894 * Force a quiescent state for RCU, which, because there is no preemptible
895 * RCU, becomes the same as rcu-sched.
897 void rcu_force_quiescent_state(void)
899 rcu_sched_force_quiescent_state();
901 EXPORT_SYMBOL_GPL(rcu_force_quiescent_state);
904 * Because preemptible RCU does not exist, we never have to check for
905 * CPUs being in quiescent states.
907 static void rcu_preempt_note_context_switch(int cpu)
912 * Because preemptible RCU does not exist, there are never any preempted
915 static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp)
920 #ifdef CONFIG_HOTPLUG_CPU
922 /* Because preemptible RCU does not exist, no quieting of tasks. */
923 static void rcu_report_unblock_qs_rnp(struct rcu_node *rnp, unsigned long flags)
925 raw_spin_unlock_irqrestore(&rnp->lock, flags);
928 #endif /* #ifdef CONFIG_HOTPLUG_CPU */
931 * Because preemptible RCU does not exist, we never have to check for
932 * tasks blocked within RCU read-side critical sections.
934 static void rcu_print_detail_task_stall(struct rcu_state *rsp)
939 * Because preemptible RCU does not exist, we never have to check for
940 * tasks blocked within RCU read-side critical sections.
942 static void rcu_print_task_stall(struct rcu_node *rnp)
947 * Because preemptible RCU does not exist, there is no need to suppress
948 * its CPU stall warnings.
950 static void rcu_preempt_stall_reset(void)
955 * Because there is no preemptible RCU, there can be no readers blocked,
956 * so there is no need to check for blocked tasks. So check only for
957 * bogus qsmask values.
959 static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp)
961 WARN_ON_ONCE(rnp->qsmask);
964 #ifdef CONFIG_HOTPLUG_CPU
967 * Because preemptible RCU does not exist, it never needs to migrate
968 * tasks that were blocked within RCU read-side critical sections, and
969 * such non-existent tasks cannot possibly have been blocking the current
972 static int rcu_preempt_offline_tasks(struct rcu_state *rsp,
973 struct rcu_node *rnp,
974 struct rcu_data *rdp)
980 * Because preemptible RCU does not exist, it never needs CPU-offline
983 static void rcu_preempt_offline_cpu(int cpu)
987 #endif /* #ifdef CONFIG_HOTPLUG_CPU */
990 * Because preemptible RCU does not exist, it never has any callbacks
993 static void rcu_preempt_check_callbacks(int cpu)
998 * Because preemptible RCU does not exist, it never has any callbacks
1001 static void rcu_preempt_process_callbacks(void)
1005 static void rcu_preempt_do_callbacks(void)
1010 * Wait for an rcu-preempt grace period, but make it happen quickly.
1011 * But because preemptible RCU does not exist, map to rcu-sched.
1013 void synchronize_rcu_expedited(void)
1015 synchronize_sched_expedited();
1017 EXPORT_SYMBOL_GPL(synchronize_rcu_expedited);
1019 #ifdef CONFIG_HOTPLUG_CPU
1022 * Because preemptible RCU does not exist, there is never any need to
1023 * report on tasks preempted in RCU read-side critical sections during
1024 * expedited RCU grace periods.
1026 static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp)
1031 #endif /* #ifdef CONFIG_HOTPLUG_CPU */
1034 * Because preemptible RCU does not exist, it never has any work to do.
1036 static int rcu_preempt_pending(int cpu)
1042 * Because preemptible RCU does not exist, it never needs any CPU.
1044 static int rcu_preempt_needs_cpu(int cpu)
1050 * Because preemptible RCU does not exist, rcu_barrier() is just
1051 * another name for rcu_barrier_sched().
1053 void rcu_barrier(void)
1055 rcu_barrier_sched();
1057 EXPORT_SYMBOL_GPL(rcu_barrier);
1060 * Because preemptible RCU does not exist, there is no per-CPU
1061 * data to initialize.
1063 static void __cpuinit rcu_preempt_init_percpu_data(int cpu)
1068 * Because there is no preemptible RCU, there are no callbacks to move.
1070 static void rcu_preempt_send_cbs_to_online(void)
1075 * Because preemptible RCU does not exist, it need not be initialized.
1077 static void __init __rcu_init_preempt(void)
1081 #endif /* #else #ifdef CONFIG_TREE_PREEMPT_RCU */
1083 #ifdef CONFIG_RCU_BOOST
1085 #include "rtmutex_common.h"
1087 #ifdef CONFIG_RCU_TRACE
1089 static void rcu_initiate_boost_trace(struct rcu_node *rnp)
1091 if (list_empty(&rnp->blkd_tasks))
1092 rnp->n_balk_blkd_tasks++;
1093 else if (rnp->exp_tasks == NULL && rnp->gp_tasks == NULL)
1094 rnp->n_balk_exp_gp_tasks++;
1095 else if (rnp->gp_tasks != NULL && rnp->boost_tasks != NULL)
1096 rnp->n_balk_boost_tasks++;
1097 else if (rnp->gp_tasks != NULL && rnp->qsmask != 0)
1098 rnp->n_balk_notblocked++;
1099 else if (rnp->gp_tasks != NULL &&
1100 ULONG_CMP_LT(jiffies, rnp->boost_time))
1101 rnp->n_balk_notyet++;
1106 #else /* #ifdef CONFIG_RCU_TRACE */
1108 static void rcu_initiate_boost_trace(struct rcu_node *rnp)
1112 #endif /* #else #ifdef CONFIG_RCU_TRACE */
1115 * Carry out RCU priority boosting on the task indicated by ->exp_tasks
1116 * or ->boost_tasks, advancing the pointer to the next task in the
1117 * ->blkd_tasks list.
1119 * Note that irqs must be enabled: boosting the task can block.
1120 * Returns 1 if there are more tasks needing to be boosted.
1122 static int rcu_boost(struct rcu_node *rnp)
1124 unsigned long flags;
1125 struct rt_mutex mtx;
1126 struct task_struct *t;
1127 struct list_head *tb;
1129 if (rnp->exp_tasks == NULL && rnp->boost_tasks == NULL)
1130 return 0; /* Nothing left to boost. */
1132 raw_spin_lock_irqsave(&rnp->lock, flags);
1135 * Recheck under the lock: all tasks in need of boosting
1136 * might exit their RCU read-side critical sections on their own.
1138 if (rnp->exp_tasks == NULL && rnp->boost_tasks == NULL) {
1139 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1144 * Preferentially boost tasks blocking expedited grace periods.
1145 * This cannot starve the normal grace periods because a second
1146 * expedited grace period must boost all blocked tasks, including
1147 * those blocking the pre-existing normal grace period.
1149 if (rnp->exp_tasks != NULL) {
1150 tb = rnp->exp_tasks;
1151 rnp->n_exp_boosts++;
1153 tb = rnp->boost_tasks;
1154 rnp->n_normal_boosts++;
1156 rnp->n_tasks_boosted++;
1159 * We boost task t by manufacturing an rt_mutex that appears to
1160 * be held by task t. We leave a pointer to that rt_mutex where
1161 * task t can find it, and task t will release the mutex when it
1162 * exits its outermost RCU read-side critical section. Then
1163 * simply acquiring this artificial rt_mutex will boost task
1164 * t's priority. (Thanks to tglx for suggesting this approach!)
1166 * Note that task t must acquire rnp->lock to remove itself from
1167 * the ->blkd_tasks list, which it will do from exit() if from
1168 * nowhere else. We therefore are guaranteed that task t will
1169 * stay around at least until we drop rnp->lock. Note that
1170 * rnp->lock also resolves races between our priority boosting
1171 * and task t's exiting its outermost RCU read-side critical
1174 t = container_of(tb, struct task_struct, rcu_node_entry);
1175 rt_mutex_init_proxy_locked(&mtx, t);
1176 t->rcu_boost_mutex = &mtx;
1177 t->rcu_read_unlock_special |= RCU_READ_UNLOCK_BOOSTED;
1178 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1179 rt_mutex_lock(&mtx); /* Side effect: boosts task t's priority. */
1180 rt_mutex_unlock(&mtx); /* Keep lockdep happy. */
1182 return rnp->exp_tasks != NULL || rnp->boost_tasks != NULL;
1186 * Timer handler to initiate waking up of boost kthreads that
1187 * have yielded the CPU due to excessive numbers of tasks to
1188 * boost. We wake up the per-rcu_node kthread, which in turn
1189 * will wake up the booster kthread.
1191 static void rcu_boost_kthread_timer(unsigned long arg)
1193 invoke_rcu_node_kthread((struct rcu_node *)arg);
1197 * Priority-boosting kthread. One per leaf rcu_node and one for the
1200 static int rcu_boost_kthread(void *arg)
1202 struct rcu_node *rnp = (struct rcu_node *)arg;
1207 rnp->boost_kthread_status = RCU_KTHREAD_WAITING;
1208 rcu_wait(rnp->boost_tasks || rnp->exp_tasks);
1209 rnp->boost_kthread_status = RCU_KTHREAD_RUNNING;
1210 more2boost = rcu_boost(rnp);
1216 rcu_yield(rcu_boost_kthread_timer, (unsigned long)rnp);
1225 * Check to see if it is time to start boosting RCU readers that are
1226 * blocking the current grace period, and, if so, tell the per-rcu_node
1227 * kthread to start boosting them. If there is an expedited grace
1228 * period in progress, it is always time to boost.
1230 * The caller must hold rnp->lock, which this function releases,
1231 * but irqs remain disabled. The ->boost_kthread_task is immortal,
1232 * so we don't need to worry about it going away.
1234 static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags)
1236 struct task_struct *t;
1238 if (!rcu_preempt_blocked_readers_cgp(rnp) && rnp->exp_tasks == NULL) {
1239 rnp->n_balk_exp_gp_tasks++;
1240 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1243 if (rnp->exp_tasks != NULL ||
1244 (rnp->gp_tasks != NULL &&
1245 rnp->boost_tasks == NULL &&
1247 ULONG_CMP_GE(jiffies, rnp->boost_time))) {
1248 if (rnp->exp_tasks == NULL)
1249 rnp->boost_tasks = rnp->gp_tasks;
1250 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1251 t = rnp->boost_kthread_task;
1255 rcu_initiate_boost_trace(rnp);
1256 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1261 * Set the affinity of the boost kthread. The CPU-hotplug locks are
1262 * held, so no one should be messing with the existence of the boost
1265 static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp,
1268 struct task_struct *t;
1270 t = rnp->boost_kthread_task;
1272 set_cpus_allowed_ptr(rnp->boost_kthread_task, cm);
1275 #define RCU_BOOST_DELAY_JIFFIES DIV_ROUND_UP(CONFIG_RCU_BOOST_DELAY * HZ, 1000)
1278 * Do priority-boost accounting for the start of a new grace period.
1280 static void rcu_preempt_boost_start_gp(struct rcu_node *rnp)
1282 rnp->boost_time = jiffies + RCU_BOOST_DELAY_JIFFIES;
1286 * Create an RCU-boost kthread for the specified node if one does not
1287 * already exist. We only create this kthread for preemptible RCU.
1288 * Returns zero if all is well, a negated errno otherwise.
1290 static int __cpuinit rcu_spawn_one_boost_kthread(struct rcu_state *rsp,
1291 struct rcu_node *rnp,
1294 unsigned long flags;
1295 struct sched_param sp;
1296 struct task_struct *t;
1298 if (&rcu_preempt_state != rsp)
1300 if (rnp->boost_kthread_task != NULL)
1302 t = kthread_create(rcu_boost_kthread, (void *)rnp,
1303 "rcub%d", rnp_index);
1306 raw_spin_lock_irqsave(&rnp->lock, flags);
1307 rnp->boost_kthread_task = t;
1308 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1309 sp.sched_priority = RCU_KTHREAD_PRIO;
1310 sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
1311 wake_up_process(t); /* get to TASK_INTERRUPTIBLE quickly. */
1315 #else /* #ifdef CONFIG_RCU_BOOST */
1317 static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags)
1319 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1322 static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp,
1327 static void rcu_preempt_boost_start_gp(struct rcu_node *rnp)
1331 static int __cpuinit rcu_spawn_one_boost_kthread(struct rcu_state *rsp,
1332 struct rcu_node *rnp,
1338 #endif /* #else #ifdef CONFIG_RCU_BOOST */
1342 void synchronize_sched_expedited(void)
1346 EXPORT_SYMBOL_GPL(synchronize_sched_expedited);
1348 #else /* #ifndef CONFIG_SMP */
1350 static atomic_t sync_sched_expedited_started = ATOMIC_INIT(0);
1351 static atomic_t sync_sched_expedited_done = ATOMIC_INIT(0);
1353 static int synchronize_sched_expedited_cpu_stop(void *data)
1356 * There must be a full memory barrier on each affected CPU
1357 * between the time that try_stop_cpus() is called and the
1358 * time that it returns.
1360 * In the current initial implementation of cpu_stop, the
1361 * above condition is already met when the control reaches
1362 * this point and the following smp_mb() is not strictly
1363 * necessary. Do smp_mb() anyway for documentation and
1364 * robustness against future implementation changes.
1366 smp_mb(); /* See above comment block. */
1371 * Wait for an rcu-sched grace period to elapse, but use "big hammer"
1372 * approach to force grace period to end quickly. This consumes
1373 * significant time on all CPUs, and is thus not recommended for
1374 * any sort of common-case code.
1376 * Note that it is illegal to call this function while holding any
1377 * lock that is acquired by a CPU-hotplug notifier. Failing to
1378 * observe this restriction will result in deadlock.
1380 * This implementation can be thought of as an application of ticket
1381 * locking to RCU, with sync_sched_expedited_started and
1382 * sync_sched_expedited_done taking on the roles of the halves
1383 * of the ticket-lock word. Each task atomically increments
1384 * sync_sched_expedited_started upon entry, snapshotting the old value,
1385 * then attempts to stop all the CPUs. If this succeeds, then each
1386 * CPU will have executed a context switch, resulting in an RCU-sched
1387 * grace period. We are then done, so we use atomic_cmpxchg() to
1388 * update sync_sched_expedited_done to match our snapshot -- but
1389 * only if someone else has not already advanced past our snapshot.
1391 * On the other hand, if try_stop_cpus() fails, we check the value
1392 * of sync_sched_expedited_done. If it has advanced past our
1393 * initial snapshot, then someone else must have forced a grace period
1394 * some time after we took our snapshot. In this case, our work is
1395 * done for us, and we can simply return. Otherwise, we try again,
1396 * but keep our initial snapshot for purposes of checking for someone
1397 * doing our work for us.
1399 * If we fail too many times in a row, we fall back to synchronize_sched().
1401 void synchronize_sched_expedited(void)
1403 int firstsnap, s, snap, trycount = 0;
1405 /* Note that atomic_inc_return() implies full memory barrier. */
1406 firstsnap = snap = atomic_inc_return(&sync_sched_expedited_started);
1410 * Each pass through the following loop attempts to force a
1411 * context switch on each CPU.
1413 while (try_stop_cpus(cpu_online_mask,
1414 synchronize_sched_expedited_cpu_stop,
1418 /* No joy, try again later. Or just synchronize_sched(). */
1419 if (trycount++ < 10)
1420 udelay(trycount * num_online_cpus());
1422 synchronize_sched();
1426 /* Check to see if someone else did our work for us. */
1427 s = atomic_read(&sync_sched_expedited_done);
1428 if (UINT_CMP_GE((unsigned)s, (unsigned)firstsnap)) {
1429 smp_mb(); /* ensure test happens before caller kfree */
1434 * Refetching sync_sched_expedited_started allows later
1435 * callers to piggyback on our grace period. We subtract
1436 * 1 to get the same token that the last incrementer got.
1437 * We retry after they started, so our grace period works
1438 * for them, and they started after our first try, so their
1439 * grace period works for us.
1442 snap = atomic_read(&sync_sched_expedited_started) - 1;
1443 smp_mb(); /* ensure read is before try_stop_cpus(). */
1447 * Everyone up to our most recent fetch is covered by our grace
1448 * period. Update the counter, but only if our work is still
1449 * relevant -- which it won't be if someone who started later
1450 * than we did beat us to the punch.
1453 s = atomic_read(&sync_sched_expedited_done);
1454 if (UINT_CMP_GE((unsigned)s, (unsigned)snap)) {
1455 smp_mb(); /* ensure test happens before caller kfree */
1458 } while (atomic_cmpxchg(&sync_sched_expedited_done, s, snap) != s);
1462 EXPORT_SYMBOL_GPL(synchronize_sched_expedited);
1464 #endif /* #else #ifndef CONFIG_SMP */
1466 #if !defined(CONFIG_RCU_FAST_NO_HZ)
1469 * Check to see if any future RCU-related work will need to be done
1470 * by the current CPU, even if none need be done immediately, returning
1471 * 1 if so. This function is part of the RCU implementation; it is -not-
1472 * an exported member of the RCU API.
1474 * Because we have preemptible RCU, just check whether this CPU needs
1475 * any flavor of RCU. Do not chew up lots of CPU cycles with preemption
1476 * disabled in a most-likely vain attempt to cause RCU not to need this CPU.
1478 int rcu_needs_cpu(int cpu)
1480 return rcu_needs_cpu_quick_check(cpu);
1484 * Check to see if we need to continue a callback-flush operations to
1485 * allow the last CPU to enter dyntick-idle mode. But fast dyntick-idle
1486 * entry is not configured, so we never do need to.
1488 static void rcu_needs_cpu_flush(void)
1492 #else /* #if !defined(CONFIG_RCU_FAST_NO_HZ) */
1494 #define RCU_NEEDS_CPU_FLUSHES 5
1495 static DEFINE_PER_CPU(int, rcu_dyntick_drain);
1496 static DEFINE_PER_CPU(unsigned long, rcu_dyntick_holdoff);
1499 * Check to see if any future RCU-related work will need to be done
1500 * by the current CPU, even if none need be done immediately, returning
1501 * 1 if so. This function is part of the RCU implementation; it is -not-
1502 * an exported member of the RCU API.
1504 * Because we are not supporting preemptible RCU, attempt to accelerate
1505 * any current grace periods so that RCU no longer needs this CPU, but
1506 * only if all other CPUs are already in dynticks-idle mode. This will
1507 * allow the CPU cores to be powered down immediately, as opposed to after
1508 * waiting many milliseconds for grace periods to elapse.
1510 * Because it is not legal to invoke rcu_process_callbacks() with irqs
1511 * disabled, we do one pass of force_quiescent_state(), then do a
1512 * invoke_rcu_cpu_kthread() to cause rcu_process_callbacks() to be invoked
1513 * later. The per-cpu rcu_dyntick_drain variable controls the sequencing.
1515 int rcu_needs_cpu(int cpu)
1521 /* Check for being in the holdoff period. */
1522 if (per_cpu(rcu_dyntick_holdoff, cpu) == jiffies)
1523 return rcu_needs_cpu_quick_check(cpu);
1525 /* Don't bother unless we are the last non-dyntick-idle CPU. */
1526 for_each_online_cpu(thatcpu) {
1529 snap = atomic_add_return(0, &per_cpu(rcu_dynticks,
1531 smp_mb(); /* Order sampling of snap with end of grace period. */
1532 if ((snap & 0x1) != 0) {
1533 per_cpu(rcu_dyntick_drain, cpu) = 0;
1534 per_cpu(rcu_dyntick_holdoff, cpu) = jiffies - 1;
1535 return rcu_needs_cpu_quick_check(cpu);
1539 /* Check and update the rcu_dyntick_drain sequencing. */
1540 if (per_cpu(rcu_dyntick_drain, cpu) <= 0) {
1541 /* First time through, initialize the counter. */
1542 per_cpu(rcu_dyntick_drain, cpu) = RCU_NEEDS_CPU_FLUSHES;
1543 } else if (--per_cpu(rcu_dyntick_drain, cpu) <= 0) {
1544 /* We have hit the limit, so time to give up. */
1545 per_cpu(rcu_dyntick_holdoff, cpu) = jiffies;
1546 return rcu_needs_cpu_quick_check(cpu);
1549 /* Do one step pushing remaining RCU callbacks through. */
1550 if (per_cpu(rcu_sched_data, cpu).nxtlist) {
1552 force_quiescent_state(&rcu_sched_state, 0);
1553 c = c || per_cpu(rcu_sched_data, cpu).nxtlist;
1555 if (per_cpu(rcu_bh_data, cpu).nxtlist) {
1557 force_quiescent_state(&rcu_bh_state, 0);
1558 c = c || per_cpu(rcu_bh_data, cpu).nxtlist;
1561 /* If RCU callbacks are still pending, RCU still needs this CPU. */
1563 invoke_rcu_cpu_kthread();
1568 * Check to see if we need to continue a callback-flush operations to
1569 * allow the last CPU to enter dyntick-idle mode.
1571 static void rcu_needs_cpu_flush(void)
1573 int cpu = smp_processor_id();
1574 unsigned long flags;
1576 if (per_cpu(rcu_dyntick_drain, cpu) <= 0)
1578 local_irq_save(flags);
1579 (void)rcu_needs_cpu(cpu);
1580 local_irq_restore(flags);
1583 #endif /* #else #if !defined(CONFIG_RCU_FAST_NO_HZ) */