2 * coupled.c - helper functions to enter the same idle state on multiple cpus
4 * Copyright (c) 2011 Google, Inc.
6 * Author: Colin Cross <ccross@android.com>
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
13 * This program is distributed in the hope that it will be useful, but WITHOUT
14 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
19 #include <linux/kernel.h>
20 #include <linux/cpu.h>
21 #include <linux/cpuidle.h>
22 #include <linux/mutex.h>
23 #include <linux/sched.h>
24 #include <linux/slab.h>
25 #include <linux/spinlock.h>
30 * DOC: Coupled cpuidle states
32 * On some ARM SMP SoCs (OMAP4460, Tegra 2, and probably more), the
33 * cpus cannot be independently powered down, either due to
34 * sequencing restrictions (on Tegra 2, cpu 0 must be the last to
35 * power down), or due to HW bugs (on OMAP4460, a cpu powering up
36 * will corrupt the gic state unless the other cpu runs a work
37 * around). Each cpu has a power state that it can enter without
38 * coordinating with the other cpu (usually Wait For Interrupt, or
39 * WFI), and one or more "coupled" power states that affect blocks
40 * shared between the cpus (L2 cache, interrupt controller, and
41 * sometimes the whole SoC). Entering a coupled power state must
42 * be tightly controlled on both cpus.
44 * This file implements a solution, where each cpu will wait in the
45 * WFI state until all cpus are ready to enter a coupled state, at
46 * which point the coupled state function will be called on all
47 * cpus at approximately the same time.
49 * Once all cpus are ready to enter idle, they are woken by an smp
50 * cross call. At this point, there is a chance that one of the
51 * cpus will find work to do, and choose not to enter idle. A
52 * final pass is needed to guarantee that all cpus will call the
53 * power state enter function at the same time. During this pass,
54 * each cpu will increment the ready counter, and continue once the
55 * ready counter matches the number of online coupled cpus. If any
56 * cpu exits idle, the other cpus will decrement their counter and
59 * requested_state stores the deepest coupled idle state each cpu
60 * is ready for. It is assumed that the states are indexed from
61 * shallowest (highest power, lowest exit latency) to deepest
62 * (lowest power, highest exit latency). The requested_state
63 * variable is not locked. It is only written from the cpu that
64 * it stores (or by the on/offlining cpu if that cpu is offline),
65 * and only read after all the cpus are ready for the coupled idle
66 * state are are no longer updating it.
68 * Three atomic counters are used. alive_count tracks the number
69 * of cpus in the coupled set that are currently or soon will be
70 * online. waiting_count tracks the number of cpus that are in
71 * the waiting loop, in the ready loop, or in the coupled idle state.
72 * ready_count tracks the number of cpus that are in the ready loop
73 * or in the coupled idle state.
75 * To use coupled cpuidle states, a cpuidle driver must:
77 * Set struct cpuidle_device.coupled_cpus to the mask of all
78 * coupled cpus, usually the same as cpu_possible_mask if all cpus
79 * are part of the same cluster. The coupled_cpus mask must be
80 * set in the struct cpuidle_device for each cpu.
82 * Set struct cpuidle_device.safe_state to a state that is not a
83 * coupled state. This is usually WFI.
85 * Set CPUIDLE_FLAG_COUPLED in struct cpuidle_state.flags for each
86 * state that affects multiple cpus.
88 * Provide a struct cpuidle_state.enter function for each state
89 * that affects multiple cpus. This function is guaranteed to be
90 * called on all cpus at approximately the same time. The driver
91 * should ensure that the cpus all abort together if any cpu tries
92 * to abort once the function is called. The function should return
93 * with interrupts still disabled.
97 * struct cpuidle_coupled - data for set of cpus that share a coupled idle state
98 * @coupled_cpus: mask of cpus that are part of the coupled set
99 * @requested_state: array of requested states for cpus in the coupled set
100 * @ready_waiting_counts: combined count of cpus in ready or waiting loops
101 * @online_count: count of cpus that are online
102 * @refcnt: reference count of cpuidle devices that are using this struct
103 * @prevent: flag to prevent coupled idle while a cpu is hotplugging
105 struct cpuidle_coupled {
106 cpumask_t coupled_cpus;
107 int requested_state[NR_CPUS];
108 atomic_t ready_waiting_counts;
109 atomic_t abort_barrier;
115 #define WAITING_BITS 16
116 #define MAX_WAITING_CPUS (1 << WAITING_BITS)
117 #define WAITING_MASK (MAX_WAITING_CPUS - 1)
118 #define READY_MASK (~WAITING_MASK)
120 #define CPUIDLE_COUPLED_NOT_IDLE (-1)
122 static DEFINE_MUTEX(cpuidle_coupled_lock);
123 static DEFINE_PER_CPU(struct call_single_data, cpuidle_coupled_poke_cb);
126 * The cpuidle_coupled_poke_pending mask is used to avoid calling
127 * __smp_call_function_single with the per cpu call_single_data struct already
128 * in use. This prevents a deadlock where two cpus are waiting for each others
129 * call_single_data struct to be available
131 static cpumask_t cpuidle_coupled_poke_pending;
134 * The cpuidle_coupled_poked mask is used to ensure that each cpu has been poked
135 * once to minimize entering the ready loop with a poke pending, which would
136 * require aborting and retrying.
138 static cpumask_t cpuidle_coupled_poked;
141 * cpuidle_coupled_parallel_barrier - synchronize all online coupled cpus
142 * @dev: cpuidle_device of the calling cpu
143 * @a: atomic variable to hold the barrier
145 * No caller to this function will return from this function until all online
146 * cpus in the same coupled group have called this function. Once any caller
147 * has returned from this function, the barrier is immediately available for
150 * The atomic variable a must be initialized to 0 before any cpu calls
151 * this function, will be reset to 0 before any cpu returns from this function.
153 * Must only be called from within a coupled idle state handler
154 * (state.enter when state.flags has CPUIDLE_FLAG_COUPLED set).
156 * Provides full smp barrier semantics before and after calling.
158 void cpuidle_coupled_parallel_barrier(struct cpuidle_device *dev, atomic_t *a)
160 int n = dev->coupled->online_count;
162 smp_mb__before_atomic_inc();
165 while (atomic_read(a) < n)
168 if (atomic_inc_return(a) == n * 2) {
173 while (atomic_read(a) > n)
178 * cpuidle_state_is_coupled - check if a state is part of a coupled set
179 * @dev: struct cpuidle_device for the current cpu
180 * @drv: struct cpuidle_driver for the platform
181 * @state: index of the target state in drv->states
183 * Returns true if the target state is coupled with cpus besides this one
185 bool cpuidle_state_is_coupled(struct cpuidle_device *dev,
186 struct cpuidle_driver *drv, int state)
188 return drv->states[state].flags & CPUIDLE_FLAG_COUPLED;
192 * cpuidle_coupled_set_ready - mark a cpu as ready
193 * @coupled: the struct coupled that contains the current cpu
195 static inline void cpuidle_coupled_set_ready(struct cpuidle_coupled *coupled)
197 atomic_add(MAX_WAITING_CPUS, &coupled->ready_waiting_counts);
201 * cpuidle_coupled_set_not_ready - mark a cpu as not ready
202 * @coupled: the struct coupled that contains the current cpu
204 * Decrements the ready counter, unless the ready (and thus the waiting) counter
205 * is equal to the number of online cpus. Prevents a race where one cpu
206 * decrements the waiting counter and then re-increments it just before another
207 * cpu has decremented its ready counter, leading to the ready counter going
208 * down from the number of online cpus without going through the coupled idle
211 * Returns 0 if the counter was decremented successfully, -EINVAL if the ready
212 * counter was equal to the number of online cpus.
215 inline int cpuidle_coupled_set_not_ready(struct cpuidle_coupled *coupled)
220 all = coupled->online_count | (coupled->online_count << WAITING_BITS);
221 ret = atomic_add_unless(&coupled->ready_waiting_counts,
222 -MAX_WAITING_CPUS, all);
224 return ret ? 0 : -EINVAL;
228 * cpuidle_coupled_no_cpus_ready - check if no cpus in a coupled set are ready
229 * @coupled: the struct coupled that contains the current cpu
231 * Returns true if all of the cpus in a coupled set are out of the ready loop.
233 static inline int cpuidle_coupled_no_cpus_ready(struct cpuidle_coupled *coupled)
235 int r = atomic_read(&coupled->ready_waiting_counts) >> WAITING_BITS;
240 * cpuidle_coupled_cpus_ready - check if all cpus in a coupled set are ready
241 * @coupled: the struct coupled that contains the current cpu
243 * Returns true if all cpus coupled to this target state are in the ready loop
245 static inline bool cpuidle_coupled_cpus_ready(struct cpuidle_coupled *coupled)
247 int r = atomic_read(&coupled->ready_waiting_counts) >> WAITING_BITS;
248 return r == coupled->online_count;
252 * cpuidle_coupled_cpus_waiting - check if all cpus in a coupled set are waiting
253 * @coupled: the struct coupled that contains the current cpu
255 * Returns true if all cpus coupled to this target state are in the wait loop
257 static inline bool cpuidle_coupled_cpus_waiting(struct cpuidle_coupled *coupled)
259 int w = atomic_read(&coupled->ready_waiting_counts) & WAITING_MASK;
260 return w == coupled->online_count;
264 * cpuidle_coupled_no_cpus_waiting - check if no cpus in coupled set are waiting
265 * @coupled: the struct coupled that contains the current cpu
267 * Returns true if all of the cpus in a coupled set are out of the waiting loop.
269 static inline int cpuidle_coupled_no_cpus_waiting(struct cpuidle_coupled *coupled)
271 int w = atomic_read(&coupled->ready_waiting_counts) & WAITING_MASK;
276 * cpuidle_coupled_get_state - determine the deepest idle state
277 * @dev: struct cpuidle_device for this cpu
278 * @coupled: the struct coupled that contains the current cpu
280 * Returns the deepest idle state that all coupled cpus can enter
282 static inline int cpuidle_coupled_get_state(struct cpuidle_device *dev,
283 struct cpuidle_coupled *coupled)
289 * Read barrier ensures that read of requested_state is ordered after
290 * reads of ready_count. Matches the write barriers
291 * cpuidle_set_state_waiting.
295 for_each_cpu_mask(i, coupled->coupled_cpus)
296 if (cpu_online(i) && coupled->requested_state[i] < state)
297 state = coupled->requested_state[i];
302 static void cpuidle_coupled_handle_poke(void *info)
304 int cpu = (unsigned long)info;
305 cpumask_set_cpu(cpu, &cpuidle_coupled_poked);
306 cpumask_clear_cpu(cpu, &cpuidle_coupled_poke_pending);
310 * cpuidle_coupled_poke - wake up a cpu that may be waiting
313 * Ensures that the target cpu exits it's waiting idle state (if it is in it)
314 * and will see updates to waiting_count before it re-enters it's waiting idle
317 * If cpuidle_coupled_poked_mask is already set for the target cpu, that cpu
318 * either has or will soon have a pending IPI that will wake it out of idle,
319 * or it is currently processing the IPI and is not in idle.
321 static void cpuidle_coupled_poke(int cpu)
323 struct call_single_data *csd = &per_cpu(cpuidle_coupled_poke_cb, cpu);
325 if (!cpumask_test_and_set_cpu(cpu, &cpuidle_coupled_poke_pending))
326 __smp_call_function_single(cpu, csd, 0);
330 * cpuidle_coupled_poke_others - wake up all other cpus that may be waiting
331 * @dev: struct cpuidle_device for this cpu
332 * @coupled: the struct coupled that contains the current cpu
334 * Calls cpuidle_coupled_poke on all other online cpus.
336 static void cpuidle_coupled_poke_others(int this_cpu,
337 struct cpuidle_coupled *coupled)
341 for_each_cpu_mask(cpu, coupled->coupled_cpus)
342 if (cpu != this_cpu && cpu_online(cpu))
343 cpuidle_coupled_poke(cpu);
347 * cpuidle_coupled_set_waiting - mark this cpu as in the wait loop
348 * @dev: struct cpuidle_device for this cpu
349 * @coupled: the struct coupled that contains the current cpu
350 * @next_state: the index in drv->states of the requested state for this cpu
352 * Updates the requested idle state for the specified cpuidle device.
353 * Returns the number of waiting cpus.
355 static int cpuidle_coupled_set_waiting(int cpu,
356 struct cpuidle_coupled *coupled, int next_state)
358 coupled->requested_state[cpu] = next_state;
361 * The atomic_inc_return provides a write barrier to order the write
362 * to requested_state with the later write that increments ready_count.
364 return atomic_inc_return(&coupled->ready_waiting_counts) & WAITING_MASK;
368 * cpuidle_coupled_set_not_waiting - mark this cpu as leaving the wait loop
369 * @dev: struct cpuidle_device for this cpu
370 * @coupled: the struct coupled that contains the current cpu
372 * Removes the requested idle state for the specified cpuidle device.
374 static void cpuidle_coupled_set_not_waiting(int cpu,
375 struct cpuidle_coupled *coupled)
378 * Decrementing waiting count can race with incrementing it in
379 * cpuidle_coupled_set_waiting, but that's OK. Worst case, some
380 * cpus will increment ready_count and then spin until they
381 * notice that this cpu has cleared it's requested_state.
383 atomic_dec(&coupled->ready_waiting_counts);
385 coupled->requested_state[cpu] = CPUIDLE_COUPLED_NOT_IDLE;
389 * cpuidle_coupled_set_done - mark this cpu as leaving the ready loop
390 * @cpu: the current cpu
391 * @coupled: the struct coupled that contains the current cpu
393 * Marks this cpu as no longer in the ready and waiting loops. Decrements
394 * the waiting count first to prevent another cpu looping back in and seeing
395 * this cpu as waiting just before it exits idle.
397 static void cpuidle_coupled_set_done(int cpu, struct cpuidle_coupled *coupled)
399 cpuidle_coupled_set_not_waiting(cpu, coupled);
400 atomic_sub(MAX_WAITING_CPUS, &coupled->ready_waiting_counts);
404 * cpuidle_coupled_clear_pokes - spin until the poke interrupt is processed
407 * Turns on interrupts and spins until any outstanding poke interrupts have
408 * been processed and the poke bit has been cleared.
410 * Other interrupts may also be processed while interrupts are enabled, so
411 * need_resched() must be tested after turning interrupts off again to make sure
412 * the interrupt didn't schedule work that should take the cpu out of idle.
414 * Returns 0 if need_resched was false, -EINTR if need_resched was true.
416 static int cpuidle_coupled_clear_pokes(int cpu)
419 while (cpumask_test_cpu(cpu, &cpuidle_coupled_poke_pending))
423 return need_resched() ? -EINTR : 0;
426 static bool cpuidle_coupled_any_pokes_pending(struct cpuidle_coupled *coupled)
431 cpumask_and(&cpus, cpu_online_mask, &coupled->coupled_cpus);
432 ret = cpumask_and(&cpus, &cpuidle_coupled_poke_pending, &cpus);
438 * cpuidle_enter_state_coupled - attempt to enter a state with coupled cpus
439 * @dev: struct cpuidle_device for the current cpu
440 * @drv: struct cpuidle_driver for the platform
441 * @next_state: index of the requested state in drv->states
443 * Coordinate with coupled cpus to enter the target state. This is a two
444 * stage process. In the first stage, the cpus are operating independently,
445 * and may call into cpuidle_enter_state_coupled at completely different times.
446 * To save as much power as possible, the first cpus to call this function will
447 * go to an intermediate state (the cpuidle_device's safe state), and wait for
448 * all the other cpus to call this function. Once all coupled cpus are idle,
449 * the second stage will start. Each coupled cpu will spin until all cpus have
450 * guaranteed that they will call the target_state.
452 * This function must be called with interrupts disabled. It may enable
453 * interrupts while preparing for idle, and it will always return with
454 * interrupts enabled.
456 int cpuidle_enter_state_coupled(struct cpuidle_device *dev,
457 struct cpuidle_driver *drv, int next_state)
459 int entered_state = -1;
460 struct cpuidle_coupled *coupled = dev->coupled;
466 while (coupled->prevent) {
467 if (cpuidle_coupled_clear_pokes(dev->cpu)) {
469 return entered_state;
471 entered_state = cpuidle_enter_state(dev, drv,
472 dev->safe_state_index);
475 /* Read barrier ensures online_count is read after prevent is cleared */
479 cpumask_clear_cpu(dev->cpu, &cpuidle_coupled_poked);
481 w = cpuidle_coupled_set_waiting(dev->cpu, coupled, next_state);
483 * If this is the last cpu to enter the waiting state, poke
484 * all the other cpus out of their waiting state so they can
485 * enter a deeper state. This can race with one of the cpus
486 * exiting the waiting state due to an interrupt and
487 * decrementing waiting_count, see comment below.
489 if (w == coupled->online_count) {
490 cpumask_set_cpu(dev->cpu, &cpuidle_coupled_poked);
491 cpuidle_coupled_poke_others(dev->cpu, coupled);
496 * Wait for all coupled cpus to be idle, using the deepest state
497 * allowed for a single cpu. If this was not the poking cpu, wait
498 * for at least one poke before leaving to avoid a race where
499 * two cpus could arrive at the waiting loop at the same time,
500 * but the first of the two to arrive could skip the loop without
501 * processing the pokes from the last to arrive.
503 while (!cpuidle_coupled_cpus_waiting(coupled) ||
504 !cpumask_test_cpu(dev->cpu, &cpuidle_coupled_poked)) {
505 if (cpuidle_coupled_clear_pokes(dev->cpu)) {
506 cpuidle_coupled_set_not_waiting(dev->cpu, coupled);
510 if (coupled->prevent) {
511 cpuidle_coupled_set_not_waiting(dev->cpu, coupled);
515 entered_state = cpuidle_enter_state(dev, drv,
516 dev->safe_state_index);
519 if (cpuidle_coupled_clear_pokes(dev->cpu)) {
520 cpuidle_coupled_set_not_waiting(dev->cpu, coupled);
525 * Make sure final poke status for this cpu is visible before setting
531 * All coupled cpus are probably idle. There is a small chance that
532 * one of the other cpus just became active. Increment the ready count,
533 * and spin until all coupled cpus have incremented the counter. Once a
534 * cpu has incremented the ready counter, it cannot abort idle and must
535 * spin until either all cpus have incremented the ready counter, or
536 * another cpu leaves idle and decrements the waiting counter.
539 cpuidle_coupled_set_ready(coupled);
540 while (!cpuidle_coupled_cpus_ready(coupled)) {
541 /* Check if any other cpus bailed out of idle. */
542 if (!cpuidle_coupled_cpus_waiting(coupled))
543 if (!cpuidle_coupled_set_not_ready(coupled))
550 * Make sure read of all cpus ready is done before reading pending pokes
555 * There is a small chance that a cpu left and reentered idle after this
556 * cpu saw that all cpus were waiting. The cpu that reentered idle will
557 * have sent this cpu a poke, which will still be pending after the
558 * ready loop. The pending interrupt may be lost by the interrupt
559 * controller when entering the deep idle state. It's not possible to
560 * clear a pending interrupt without turning interrupts on and handling
561 * it, and it's too late to turn on interrupts here, so reset the
562 * coupled idle state of all cpus and retry.
564 if (cpuidle_coupled_any_pokes_pending(coupled)) {
565 cpuidle_coupled_set_done(dev->cpu, coupled);
566 /* Wait for all cpus to see the pending pokes */
567 cpuidle_coupled_parallel_barrier(dev, &coupled->abort_barrier);
571 /* all cpus have acked the coupled state */
572 next_state = cpuidle_coupled_get_state(dev, coupled);
574 entered_state = cpuidle_enter_state(dev, drv, next_state);
576 cpuidle_coupled_set_done(dev->cpu, coupled);
580 * Normal cpuidle states are expected to return with irqs enabled.
581 * That leads to an inefficiency where a cpu receiving an interrupt
582 * that brings it out of idle will process that interrupt before
583 * exiting the idle enter function and decrementing ready_count. All
584 * other cpus will need to spin waiting for the cpu that is processing
585 * the interrupt. If the driver returns with interrupts disabled,
586 * all other cpus will loop back into the safe idle state instead of
587 * spinning, saving power.
589 * Calling local_irq_enable here allows coupled states to return with
590 * interrupts disabled, but won't cause problems for drivers that
591 * exit with interrupts enabled.
596 * Wait until all coupled cpus have exited idle. There is no risk that
597 * a cpu exits and re-enters the ready state because this cpu has
598 * already decremented its waiting_count.
600 while (!cpuidle_coupled_no_cpus_ready(coupled))
603 return entered_state;
606 static void cpuidle_coupled_update_online_cpus(struct cpuidle_coupled *coupled)
609 cpumask_and(&cpus, cpu_online_mask, &coupled->coupled_cpus);
610 coupled->online_count = cpumask_weight(&cpus);
614 * cpuidle_coupled_register_device - register a coupled cpuidle device
615 * @dev: struct cpuidle_device for the current cpu
617 * Called from cpuidle_register_device to handle coupled idle init. Finds the
618 * cpuidle_coupled struct for this set of coupled cpus, or creates one if none
621 int cpuidle_coupled_register_device(struct cpuidle_device *dev)
624 struct cpuidle_device *other_dev;
625 struct call_single_data *csd;
626 struct cpuidle_coupled *coupled;
628 if (cpumask_empty(&dev->coupled_cpus))
631 for_each_cpu_mask(cpu, dev->coupled_cpus) {
632 other_dev = per_cpu(cpuidle_devices, cpu);
633 if (other_dev && other_dev->coupled) {
634 coupled = other_dev->coupled;
639 /* No existing coupled info found, create a new one */
640 coupled = kzalloc(sizeof(struct cpuidle_coupled), GFP_KERNEL);
644 coupled->coupled_cpus = dev->coupled_cpus;
647 dev->coupled = coupled;
648 if (WARN_ON(!cpumask_equal(&dev->coupled_cpus, &coupled->coupled_cpus)))
651 cpuidle_coupled_update_online_cpus(coupled);
655 csd = &per_cpu(cpuidle_coupled_poke_cb, dev->cpu);
656 csd->func = cpuidle_coupled_handle_poke;
657 csd->info = (void *)(unsigned long)dev->cpu;
663 * cpuidle_coupled_unregister_device - unregister a coupled cpuidle device
664 * @dev: struct cpuidle_device for the current cpu
666 * Called from cpuidle_unregister_device to tear down coupled idle. Removes the
667 * cpu from the coupled idle set, and frees the cpuidle_coupled_info struct if
668 * this was the last cpu in the set.
670 void cpuidle_coupled_unregister_device(struct cpuidle_device *dev)
672 struct cpuidle_coupled *coupled = dev->coupled;
674 if (cpumask_empty(&dev->coupled_cpus))
677 if (--coupled->refcnt)
683 * cpuidle_coupled_prevent_idle - prevent cpus from entering a coupled state
684 * @coupled: the struct coupled that contains the cpu that is changing state
686 * Disables coupled cpuidle on a coupled set of cpus. Used to ensure that
687 * cpu_online_mask doesn't change while cpus are coordinating coupled idle.
689 static void cpuidle_coupled_prevent_idle(struct cpuidle_coupled *coupled)
693 /* Force all cpus out of the waiting loop. */
695 cpuidle_coupled_poke_others(cpu, coupled);
697 while (!cpuidle_coupled_no_cpus_waiting(coupled))
702 * cpuidle_coupled_allow_idle - allows cpus to enter a coupled state
703 * @coupled: the struct coupled that contains the cpu that is changing state
705 * Enables coupled cpuidle on a coupled set of cpus. Used to ensure that
706 * cpu_online_mask doesn't change while cpus are coordinating coupled idle.
708 static void cpuidle_coupled_allow_idle(struct cpuidle_coupled *coupled)
713 * Write barrier ensures readers see the new online_count when they
718 /* Force cpus out of the prevent loop. */
719 cpuidle_coupled_poke_others(cpu, coupled);
724 * cpuidle_coupled_cpu_notify - notifier called during hotplug transitions
725 * @nb: notifier block
726 * @action: hotplug transition
727 * @hcpu: target cpu number
729 * Called when a cpu is brought on or offline using hotplug. Updates the
730 * coupled cpu set appropriately
732 static int cpuidle_coupled_cpu_notify(struct notifier_block *nb,
733 unsigned long action, void *hcpu)
735 int cpu = (unsigned long)hcpu;
736 struct cpuidle_device *dev;
738 switch (action & ~CPU_TASKS_FROZEN) {
740 case CPU_DOWN_PREPARE:
743 case CPU_UP_CANCELED:
744 case CPU_DOWN_FAILED:
750 mutex_lock(&cpuidle_lock);
752 dev = per_cpu(cpuidle_devices, cpu);
753 if (!dev || !dev->coupled)
756 switch (action & ~CPU_TASKS_FROZEN) {
758 case CPU_DOWN_PREPARE:
759 cpuidle_coupled_prevent_idle(dev->coupled);
763 cpuidle_coupled_update_online_cpus(dev->coupled);
765 case CPU_UP_CANCELED:
766 case CPU_DOWN_FAILED:
767 cpuidle_coupled_allow_idle(dev->coupled);
772 mutex_unlock(&cpuidle_lock);
776 static struct notifier_block cpuidle_coupled_cpu_notifier = {
777 .notifier_call = cpuidle_coupled_cpu_notify,
780 static int __init cpuidle_coupled_init(void)
782 return register_cpu_notifier(&cpuidle_coupled_cpu_notifier);
784 core_initcall(cpuidle_coupled_init);