2 * arch/arm/common/mcpm_entry.c -- entry point for multi-cluster PM
4 * Created by: Nicolas Pitre, March 2012
5 * Copyright: (C) 2012-2013 Linaro Limited
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
12 #include <linux/kernel.h>
13 #include <linux/init.h>
14 #include <linux/irqflags.h>
15 #include <linux/cpu_pm.h>
18 #include <asm/cacheflush.h>
19 #include <asm/idmap.h>
20 #include <asm/cputype.h>
21 #include <asm/suspend.h>
23 extern unsigned long mcpm_entry_vectors[MAX_NR_CLUSTERS][MAX_CPUS_PER_CLUSTER];
25 void mcpm_set_entry_vector(unsigned cpu, unsigned cluster, void *ptr)
27 unsigned long val = ptr ? virt_to_phys(ptr) : 0;
28 mcpm_entry_vectors[cluster][cpu] = val;
29 sync_cache_w(&mcpm_entry_vectors[cluster][cpu]);
32 extern unsigned long mcpm_entry_early_pokes[MAX_NR_CLUSTERS][MAX_CPUS_PER_CLUSTER][2];
34 void mcpm_set_early_poke(unsigned cpu, unsigned cluster,
35 unsigned long poke_phys_addr, unsigned long poke_val)
37 unsigned long *poke = &mcpm_entry_early_pokes[cluster][cpu][0];
38 poke[0] = poke_phys_addr;
40 __sync_cache_range_w(poke, 2 * sizeof(*poke));
43 static const struct mcpm_platform_ops *platform_ops;
45 int __init mcpm_platform_register(const struct mcpm_platform_ops *ops)
53 bool mcpm_is_available(void)
55 return (platform_ops) ? true : false;
59 * We can't use regular spinlocks. In the switcher case, it is possible
60 * for an outbound CPU to call power_down() after its inbound counterpart
61 * is already live using the same logical CPU number which trips lockdep
64 static arch_spinlock_t mcpm_lock = __ARCH_SPIN_LOCK_UNLOCKED;
66 static int mcpm_cpu_use_count[MAX_NR_CLUSTERS][MAX_CPUS_PER_CLUSTER];
68 static inline bool mcpm_cluster_unused(unsigned int cluster)
71 for (i = 0, cnt = 0; i < MAX_CPUS_PER_CLUSTER; i++)
72 cnt |= mcpm_cpu_use_count[cluster][i];
76 int mcpm_cpu_power_up(unsigned int cpu, unsigned int cluster)
78 bool cpu_is_down, cluster_is_down;
81 pr_debug("%s: cpu %u cluster %u\n", __func__, cpu, cluster);
83 return -EUNATCH; /* try not to shadow power_up errors */
87 * Since this is called with IRQs enabled, and no arch_spin_lock_irq
88 * variant exists, we need to disable IRQs manually here.
91 arch_spin_lock(&mcpm_lock);
93 cpu_is_down = !mcpm_cpu_use_count[cluster][cpu];
94 cluster_is_down = mcpm_cluster_unused(cluster);
96 mcpm_cpu_use_count[cluster][cpu]++;
98 * The only possible values are:
101 * 2 = CPU requested to be up before it had a chance
102 * to actually make itself down.
103 * Any other value is a bug.
105 BUG_ON(mcpm_cpu_use_count[cluster][cpu] != 1 &&
106 mcpm_cpu_use_count[cluster][cpu] != 2);
109 ret = platform_ops->cluster_powerup(cluster);
110 if (cpu_is_down && !ret)
111 ret = platform_ops->cpu_powerup(cpu, cluster);
113 arch_spin_unlock(&mcpm_lock);
118 typedef void (*phys_reset_t)(unsigned long);
120 void mcpm_cpu_power_down(void)
122 unsigned int mpidr, cpu, cluster;
123 bool cpu_going_down, last_man;
124 phys_reset_t phys_reset;
126 mpidr = read_cpuid_mpidr();
127 cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
128 cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
129 pr_debug("%s: cpu %u cluster %u\n", __func__, cpu, cluster);
130 if (WARN_ON_ONCE(!platform_ops))
132 BUG_ON(!irqs_disabled());
134 setup_mm_for_reboot();
136 __mcpm_cpu_going_down(cpu, cluster);
137 arch_spin_lock(&mcpm_lock);
138 BUG_ON(__mcpm_cluster_state(cluster) != CLUSTER_UP);
140 mcpm_cpu_use_count[cluster][cpu]--;
141 BUG_ON(mcpm_cpu_use_count[cluster][cpu] != 0 &&
142 mcpm_cpu_use_count[cluster][cpu] != 1);
143 cpu_going_down = !mcpm_cpu_use_count[cluster][cpu];
144 last_man = mcpm_cluster_unused(cluster);
146 if (last_man && __mcpm_outbound_enter_critical(cpu, cluster)) {
147 platform_ops->cpu_powerdown_prepare(cpu, cluster);
148 platform_ops->cluster_powerdown_prepare(cluster);
149 arch_spin_unlock(&mcpm_lock);
150 platform_ops->cluster_cache_disable();
151 __mcpm_outbound_leave_critical(cluster, CLUSTER_DOWN);
154 platform_ops->cpu_powerdown_prepare(cpu, cluster);
155 arch_spin_unlock(&mcpm_lock);
157 * If cpu_going_down is false here, that means a power_up
158 * request raced ahead of us. Even if we do not want to
159 * shut this CPU down, the caller still expects execution
160 * to return through the system resume entry path, like
161 * when the WFI is aborted due to a new IRQ or the like..
162 * So let's continue with cache cleaning in all cases.
164 platform_ops->cpu_cache_disable();
167 __mcpm_cpu_down(cpu, cluster);
169 /* Now we are prepared for power-down, do it: */
174 * It is possible for a power_up request to happen concurrently
175 * with a power_down request for the same CPU. In this case the
176 * CPU might not be able to actually enter a powered down state
177 * with the WFI instruction if the power_up request has removed
178 * the required reset condition. We must perform a re-entry in
179 * the kernel as if the power_up method just had deasserted reset
182 phys_reset = (phys_reset_t)(unsigned long)virt_to_phys(cpu_reset);
183 phys_reset(virt_to_phys(mcpm_entry_point));
185 /* should never get here */
189 int mcpm_wait_for_cpu_powerdown(unsigned int cpu, unsigned int cluster)
193 if (WARN_ON_ONCE(!platform_ops || !platform_ops->wait_for_powerdown))
196 ret = platform_ops->wait_for_powerdown(cpu, cluster);
198 pr_warn("%s: cpu %u, cluster %u failed to power down (%d)\n",
199 __func__, cpu, cluster, ret);
204 void mcpm_cpu_suspend(u64 expected_residency)
206 if (WARN_ON_ONCE(!platform_ops))
209 /* Some platforms might have to enable special resume modes, etc. */
210 if (platform_ops->cpu_suspend_prepare) {
211 unsigned int mpidr = read_cpuid_mpidr();
212 unsigned int cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
213 unsigned int cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
214 arch_spin_lock(&mcpm_lock);
215 platform_ops->cpu_suspend_prepare(cpu, cluster);
216 arch_spin_unlock(&mcpm_lock);
218 mcpm_cpu_power_down();
221 int mcpm_cpu_powered_up(void)
223 unsigned int mpidr, cpu, cluster;
224 bool cpu_was_down, first_man;
230 mpidr = read_cpuid_mpidr();
231 cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
232 cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
233 local_irq_save(flags);
234 arch_spin_lock(&mcpm_lock);
236 cpu_was_down = !mcpm_cpu_use_count[cluster][cpu];
237 first_man = mcpm_cluster_unused(cluster);
239 if (first_man && platform_ops->cluster_is_up)
240 platform_ops->cluster_is_up(cluster);
242 mcpm_cpu_use_count[cluster][cpu] = 1;
243 if (platform_ops->cpu_is_up)
244 platform_ops->cpu_is_up(cpu, cluster);
246 arch_spin_unlock(&mcpm_lock);
247 local_irq_restore(flags);
252 #ifdef CONFIG_ARM_CPU_SUSPEND
254 static int __init nocache_trampoline(unsigned long _arg)
256 void (*cache_disable)(void) = (void *)_arg;
257 unsigned int mpidr = read_cpuid_mpidr();
258 unsigned int cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
259 unsigned int cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
260 phys_reset_t phys_reset;
262 mcpm_set_entry_vector(cpu, cluster, cpu_resume);
263 setup_mm_for_reboot();
265 __mcpm_cpu_going_down(cpu, cluster);
266 BUG_ON(!__mcpm_outbound_enter_critical(cpu, cluster));
268 __mcpm_outbound_leave_critical(cluster, CLUSTER_DOWN);
269 __mcpm_cpu_down(cpu, cluster);
271 phys_reset = (phys_reset_t)(unsigned long)virt_to_phys(cpu_reset);
272 phys_reset(virt_to_phys(mcpm_entry_point));
276 int __init mcpm_loopback(void (*cache_disable)(void))
281 * We're going to soft-restart the current CPU through the
282 * low-level MCPM code by leveraging the suspend/resume
283 * infrastructure. Let's play it safe by using cpu_pm_enter()
284 * in case the CPU init code path resets the VFP or similar.
288 ret = cpu_pm_enter();
290 ret = cpu_suspend((unsigned long)cache_disable, nocache_trampoline);
296 pr_err("%s returned %d\n", __func__, ret);
302 struct sync_struct mcpm_sync;
305 * __mcpm_cpu_going_down: Indicates that the cpu is being torn down.
306 * This must be called at the point of committing to teardown of a CPU.
307 * The CPU cache (SCTRL.C bit) is expected to still be active.
309 void __mcpm_cpu_going_down(unsigned int cpu, unsigned int cluster)
311 mcpm_sync.clusters[cluster].cpus[cpu].cpu = CPU_GOING_DOWN;
312 sync_cache_w(&mcpm_sync.clusters[cluster].cpus[cpu].cpu);
316 * __mcpm_cpu_down: Indicates that cpu teardown is complete and that the
317 * cluster can be torn down without disrupting this CPU.
318 * To avoid deadlocks, this must be called before a CPU is powered down.
319 * The CPU cache (SCTRL.C bit) is expected to be off.
320 * However L2 cache might or might not be active.
322 void __mcpm_cpu_down(unsigned int cpu, unsigned int cluster)
325 mcpm_sync.clusters[cluster].cpus[cpu].cpu = CPU_DOWN;
326 sync_cache_w(&mcpm_sync.clusters[cluster].cpus[cpu].cpu);
331 * __mcpm_outbound_leave_critical: Leave the cluster teardown critical section.
332 * @state: the final state of the cluster:
333 * CLUSTER_UP: no destructive teardown was done and the cluster has been
334 * restored to the previous state (CPU cache still active); or
335 * CLUSTER_DOWN: the cluster has been torn-down, ready for power-off
336 * (CPU cache disabled, L2 cache either enabled or disabled).
338 void __mcpm_outbound_leave_critical(unsigned int cluster, int state)
341 mcpm_sync.clusters[cluster].cluster = state;
342 sync_cache_w(&mcpm_sync.clusters[cluster].cluster);
347 * __mcpm_outbound_enter_critical: Enter the cluster teardown critical section.
348 * This function should be called by the last man, after local CPU teardown
349 * is complete. CPU cache expected to be active.
352 * false: the critical section was not entered because an inbound CPU was
353 * observed, or the cluster is already being set up;
354 * true: the critical section was entered: it is now safe to tear down the
357 bool __mcpm_outbound_enter_critical(unsigned int cpu, unsigned int cluster)
360 struct mcpm_sync_struct *c = &mcpm_sync.clusters[cluster];
362 /* Warn inbound CPUs that the cluster is being torn down: */
363 c->cluster = CLUSTER_GOING_DOWN;
364 sync_cache_w(&c->cluster);
366 /* Back out if the inbound cluster is already in the critical region: */
367 sync_cache_r(&c->inbound);
368 if (c->inbound == INBOUND_COMING_UP)
372 * Wait for all CPUs to get out of the GOING_DOWN state, so that local
373 * teardown is complete on each CPU before tearing down the cluster.
375 * If any CPU has been woken up again from the DOWN state, then we
376 * shouldn't be taking the cluster down at all: abort in that case.
378 sync_cache_r(&c->cpus);
379 for (i = 0; i < MAX_CPUS_PER_CLUSTER; i++) {
386 cpustate = c->cpus[i].cpu;
387 if (cpustate != CPU_GOING_DOWN)
391 sync_cache_r(&c->cpus[i].cpu);
406 __mcpm_outbound_leave_critical(cluster, CLUSTER_UP);
410 int __mcpm_cluster_state(unsigned int cluster)
412 sync_cache_r(&mcpm_sync.clusters[cluster].cluster);
413 return mcpm_sync.clusters[cluster].cluster;
416 extern unsigned long mcpm_power_up_setup_phys;
418 int __init mcpm_sync_init(
419 void (*power_up_setup)(unsigned int affinity_level))
421 unsigned int i, j, mpidr, this_cluster;
423 BUILD_BUG_ON(MCPM_SYNC_CLUSTER_SIZE * MAX_NR_CLUSTERS != sizeof mcpm_sync);
424 BUG_ON((unsigned long)&mcpm_sync & (__CACHE_WRITEBACK_GRANULE - 1));
427 * Set initial CPU and cluster states.
428 * Only one cluster is assumed to be active at this point.
430 for (i = 0; i < MAX_NR_CLUSTERS; i++) {
431 mcpm_sync.clusters[i].cluster = CLUSTER_DOWN;
432 mcpm_sync.clusters[i].inbound = INBOUND_NOT_COMING_UP;
433 for (j = 0; j < MAX_CPUS_PER_CLUSTER; j++)
434 mcpm_sync.clusters[i].cpus[j].cpu = CPU_DOWN;
436 mpidr = read_cpuid_mpidr();
437 this_cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
438 for_each_online_cpu(i) {
439 mcpm_cpu_use_count[this_cluster][i] = 1;
440 mcpm_sync.clusters[this_cluster].cpus[i].cpu = CPU_UP;
442 mcpm_sync.clusters[this_cluster].cluster = CLUSTER_UP;
443 sync_cache_w(&mcpm_sync);
445 if (power_up_setup) {
446 mcpm_power_up_setup_phys = virt_to_phys(power_up_setup);
447 sync_cache_w(&mcpm_power_up_setup_phys);