2 * linux/drivers/cpufreq/cpufreq.c
4 * Copyright (C) 2001 Russell King
5 * (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
7 * Oct 2005 - Ashok Raj <ashok.raj@intel.com>
8 * Added handling for CPU hotplug
9 * Feb 2006 - Jacob Shin <jacob.shin@amd.com>
10 * Fix handling for CPU hotplug -- affected CPUs
12 * This program is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License version 2 as
14 * published by the Free Software Foundation.
18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
20 #include <asm/cputime.h>
21 #include <linux/kernel.h>
22 #include <linux/kernel_stat.h>
23 #include <linux/module.h>
24 #include <linux/init.h>
25 #include <linux/notifier.h>
26 #include <linux/cpufreq.h>
27 #include <linux/delay.h>
28 #include <linux/interrupt.h>
29 #include <linux/spinlock.h>
30 #include <linux/tick.h>
31 #include <linux/device.h>
32 #include <linux/slab.h>
33 #include <linux/cpu.h>
34 #include <linux/completion.h>
35 #include <linux/mutex.h>
36 #include <linux/syscore_ops.h>
37 #include <linux/suspend.h>
38 #include <linux/tick.h>
40 #include <trace/events/power.h>
43 * The "cpufreq driver" - the arch- or hardware-dependent low
44 * level driver of CPUFreq support, and its spinlock. This lock
45 * also protects the cpufreq_cpu_data array.
47 static struct cpufreq_driver *cpufreq_driver;
48 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
49 #ifdef CONFIG_HOTPLUG_CPU
50 /* This one keeps track of the previously set governor of a removed CPU */
51 static DEFINE_PER_CPU(char[CPUFREQ_NAME_LEN], cpufreq_cpu_governor);
53 static DEFINE_RWLOCK(cpufreq_driver_lock);
54 static DEFINE_MUTEX(cpufreq_governor_lock);
56 /* Flag to suspend/resume CPUFreq governors */
57 static bool cpufreq_suspended;
59 static inline bool has_target(void)
61 return cpufreq_driver->target;
65 * cpu_policy_rwsem is a per CPU reader-writer semaphore designed to cure
66 * all cpufreq/hotplug/workqueue/etc related lock issues.
68 * The rules for this semaphore:
69 * - Any routine that wants to read from the policy structure will
70 * do a down_read on this semaphore.
71 * - Any routine that will write to the policy structure and/or may take away
72 * the policy altogether (eg. CPU hotplug), will hold this lock in write
73 * mode before doing so.
76 * - Governor routines that can be called in cpufreq hotplug path should not
77 * take this sem as top level hotplug notifier handler takes this.
78 * - Lock should not be held across
79 * __cpufreq_governor(data, CPUFREQ_GOV_STOP);
81 static DEFINE_PER_CPU(int, cpufreq_policy_cpu);
82 static DEFINE_PER_CPU(struct rw_semaphore, cpu_policy_rwsem);
84 #define lock_policy_rwsem(mode, cpu) \
85 static int lock_policy_rwsem_##mode(int cpu) \
87 int policy_cpu = per_cpu(cpufreq_policy_cpu, cpu); \
88 BUG_ON(policy_cpu == -1); \
89 down_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu)); \
94 lock_policy_rwsem(read, cpu);
95 lock_policy_rwsem(write, cpu);
97 #define unlock_policy_rwsem(mode, cpu) \
98 static void unlock_policy_rwsem_##mode(int cpu) \
100 int policy_cpu = per_cpu(cpufreq_policy_cpu, cpu); \
101 BUG_ON(policy_cpu == -1); \
102 up_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu)); \
105 unlock_policy_rwsem(read, cpu);
106 unlock_policy_rwsem(write, cpu);
108 /* internal prototypes */
109 static int __cpufreq_governor(struct cpufreq_policy *policy,
111 static unsigned int __cpufreq_get(unsigned int cpu);
112 static void handle_update(struct work_struct *work);
115 * Two notifier lists: the "policy" list is involved in the
116 * validation process for a new CPU frequency policy; the
117 * "transition" list for kernel code that needs to handle
118 * changes to devices when the CPU clock speed changes.
119 * The mutex locks both lists.
121 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
122 static struct srcu_notifier_head cpufreq_transition_notifier_list;
124 static bool init_cpufreq_transition_notifier_list_called;
125 static int __init init_cpufreq_transition_notifier_list(void)
127 srcu_init_notifier_head(&cpufreq_transition_notifier_list);
128 init_cpufreq_transition_notifier_list_called = true;
131 pure_initcall(init_cpufreq_transition_notifier_list);
133 static int off __read_mostly;
134 static int cpufreq_disabled(void)
138 void disable_cpufreq(void)
142 static LIST_HEAD(cpufreq_governor_list);
143 static DEFINE_MUTEX(cpufreq_governor_mutex);
145 bool have_governor_per_policy(void)
147 return cpufreq_driver->have_governor_per_policy;
149 EXPORT_SYMBOL_GPL(have_governor_per_policy);
151 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
153 if (have_governor_per_policy())
154 return &policy->kobj;
156 return cpufreq_global_kobject;
158 EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
160 static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
166 cur_wall_time = jiffies64_to_cputime64(get_jiffies_64());
168 busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER];
169 busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM];
170 busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ];
171 busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ];
172 busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL];
173 busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE];
175 idle_time = cur_wall_time - busy_time;
177 *wall = cputime_to_usecs(cur_wall_time);
179 return cputime_to_usecs(idle_time);
182 u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
184 u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);
186 if (idle_time == -1ULL)
187 return get_cpu_idle_time_jiffy(cpu, wall);
189 idle_time += get_cpu_iowait_time_us(cpu, wall);
193 EXPORT_SYMBOL_GPL(get_cpu_idle_time);
195 static struct cpufreq_policy *__cpufreq_cpu_get(unsigned int cpu, bool sysfs)
197 struct cpufreq_policy *data;
200 if (cpu >= nr_cpu_ids)
203 /* get the cpufreq driver */
204 read_lock_irqsave(&cpufreq_driver_lock, flags);
209 if (!try_module_get(cpufreq_driver->owner))
214 data = per_cpu(cpufreq_cpu_data, cpu);
217 goto err_out_put_module;
219 if (!sysfs && !kobject_get(&data->kobj))
220 goto err_out_put_module;
222 read_unlock_irqrestore(&cpufreq_driver_lock, flags);
226 module_put(cpufreq_driver->owner);
228 read_unlock_irqrestore(&cpufreq_driver_lock, flags);
233 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
235 if (cpufreq_disabled())
238 return __cpufreq_cpu_get(cpu, false);
240 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
242 static struct cpufreq_policy *cpufreq_cpu_get_sysfs(unsigned int cpu)
244 return __cpufreq_cpu_get(cpu, true);
247 static void __cpufreq_cpu_put(struct cpufreq_policy *data, bool sysfs)
250 kobject_put(&data->kobj);
251 module_put(cpufreq_driver->owner);
254 void cpufreq_cpu_put(struct cpufreq_policy *data)
256 if (cpufreq_disabled())
259 __cpufreq_cpu_put(data, false);
261 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
263 static void cpufreq_cpu_put_sysfs(struct cpufreq_policy *data)
265 __cpufreq_cpu_put(data, true);
268 /*********************************************************************
269 * EXTERNALLY AFFECTING FREQUENCY CHANGES *
270 *********************************************************************/
273 * adjust_jiffies - adjust the system "loops_per_jiffy"
275 * This function alters the system "loops_per_jiffy" for the clock
276 * speed change. Note that loops_per_jiffy cannot be updated on SMP
277 * systems as each CPU might be scaled differently. So, use the arch
278 * per-CPU loops_per_jiffy value wherever possible.
281 static unsigned long l_p_j_ref;
282 static unsigned int l_p_j_ref_freq;
284 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
286 if (ci->flags & CPUFREQ_CONST_LOOPS)
289 if (!l_p_j_ref_freq) {
290 l_p_j_ref = loops_per_jiffy;
291 l_p_j_ref_freq = ci->old;
292 pr_debug("saving %lu as reference value for loops_per_jiffy; "
293 "freq is %u kHz\n", l_p_j_ref, l_p_j_ref_freq);
295 if ((val == CPUFREQ_POSTCHANGE && ci->old != ci->new) ||
296 (val == CPUFREQ_RESUMECHANGE || val == CPUFREQ_SUSPENDCHANGE)) {
297 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
299 pr_debug("scaling loops_per_jiffy to %lu "
300 "for frequency %u kHz\n", loops_per_jiffy, ci->new);
304 static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
311 void __cpufreq_notify_transition(struct cpufreq_policy *policy,
312 struct cpufreq_freqs *freqs, unsigned int state)
314 BUG_ON(irqs_disabled());
316 if (cpufreq_disabled())
319 freqs->flags = cpufreq_driver->flags;
320 pr_debug("notification %u of frequency transition to %u kHz\n",
325 case CPUFREQ_PRECHANGE:
326 /* detect if the driver reported a value as "old frequency"
327 * which is not equal to what the cpufreq core thinks is
330 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
331 if ((policy) && (policy->cpu == freqs->cpu) &&
332 (policy->cur) && (policy->cur != freqs->old)) {
333 pr_debug("Warning: CPU frequency is"
334 " %u, cpufreq assumed %u kHz.\n",
335 freqs->old, policy->cur);
336 freqs->old = policy->cur;
339 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
340 CPUFREQ_PRECHANGE, freqs);
341 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
344 case CPUFREQ_POSTCHANGE:
345 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
346 pr_debug("FREQ: %lu - CPU: %lu", (unsigned long)freqs->new,
347 (unsigned long)freqs->cpu);
348 trace_cpu_frequency(freqs->new, freqs->cpu);
349 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
350 CPUFREQ_POSTCHANGE, freqs);
351 if (likely(policy) && likely(policy->cpu == freqs->cpu))
352 policy->cur = freqs->new;
357 * cpufreq_notify_transition - call notifier chain and adjust_jiffies
358 * on frequency transition.
360 * This function calls the transition notifiers and the "adjust_jiffies"
361 * function. It is called twice on all CPU frequency changes that have
364 void cpufreq_notify_transition(struct cpufreq_policy *policy,
365 struct cpufreq_freqs *freqs, unsigned int state)
367 for_each_cpu(freqs->cpu, policy->cpus)
368 __cpufreq_notify_transition(policy, freqs, state);
370 EXPORT_SYMBOL_GPL(cpufreq_notify_transition);
374 /*********************************************************************
376 *********************************************************************/
378 static struct cpufreq_governor *__find_governor(const char *str_governor)
380 struct cpufreq_governor *t;
382 list_for_each_entry(t, &cpufreq_governor_list, governor_list)
383 if (!strnicmp(str_governor, t->name, CPUFREQ_NAME_LEN))
390 * cpufreq_parse_governor - parse a governor string
392 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
393 struct cpufreq_governor **governor)
400 if (cpufreq_driver->setpolicy) {
401 if (!strnicmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
402 *policy = CPUFREQ_POLICY_PERFORMANCE;
404 } else if (!strnicmp(str_governor, "powersave",
406 *policy = CPUFREQ_POLICY_POWERSAVE;
409 } else if (cpufreq_driver->target) {
410 struct cpufreq_governor *t;
412 mutex_lock(&cpufreq_governor_mutex);
414 t = __find_governor(str_governor);
419 mutex_unlock(&cpufreq_governor_mutex);
420 ret = request_module("cpufreq_%s", str_governor);
421 mutex_lock(&cpufreq_governor_mutex);
424 t = __find_governor(str_governor);
432 mutex_unlock(&cpufreq_governor_mutex);
440 * cpufreq_per_cpu_attr_read() / show_##file_name() -
441 * print out cpufreq information
443 * Write out information from cpufreq_driver->policy[cpu]; object must be
447 #define show_one(file_name, object) \
448 static ssize_t show_##file_name \
449 (struct cpufreq_policy *policy, char *buf) \
451 return sprintf(buf, "%u\n", policy->object); \
454 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
455 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
456 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
457 show_one(scaling_min_freq, min);
458 show_one(scaling_max_freq, max);
459 show_one(scaling_cur_freq, cur);
461 static int __cpufreq_set_policy(struct cpufreq_policy *data,
462 struct cpufreq_policy *policy);
465 * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
467 #define store_one(file_name, object) \
468 static ssize_t store_##file_name \
469 (struct cpufreq_policy *policy, const char *buf, size_t count) \
472 struct cpufreq_policy new_policy; \
474 ret = cpufreq_get_policy(&new_policy, policy->cpu); \
478 ret = sscanf(buf, "%u", &new_policy.object); \
482 ret = __cpufreq_set_policy(policy, &new_policy); \
483 policy->user_policy.object = policy->object; \
485 return ret ? ret : count; \
488 store_one(scaling_min_freq, min);
489 store_one(scaling_max_freq, max);
492 * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
494 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
497 unsigned int cur_freq = __cpufreq_get(policy->cpu);
499 return sprintf(buf, "<unknown>");
500 return sprintf(buf, "%u\n", cur_freq);
505 * show_scaling_governor - show the current policy for the specified CPU
507 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
509 if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
510 return sprintf(buf, "powersave\n");
511 else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
512 return sprintf(buf, "performance\n");
513 else if (policy->governor)
514 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
515 policy->governor->name);
521 * store_scaling_governor - store policy for the specified CPU
523 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
524 const char *buf, size_t count)
527 char str_governor[16];
528 struct cpufreq_policy new_policy;
530 ret = cpufreq_get_policy(&new_policy, policy->cpu);
534 ret = sscanf(buf, "%15s", str_governor);
538 if (cpufreq_parse_governor(str_governor, &new_policy.policy,
539 &new_policy.governor))
542 /* Do not use cpufreq_set_policy here or the user_policy.max
543 will be wrongly overridden */
544 ret = __cpufreq_set_policy(policy, &new_policy);
546 policy->user_policy.policy = policy->policy;
547 policy->user_policy.governor = policy->governor;
556 * show_scaling_driver - show the cpufreq driver currently loaded
558 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
560 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
564 * show_scaling_available_governors - show the available CPUfreq governors
566 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
570 struct cpufreq_governor *t;
572 if (!cpufreq_driver->target) {
573 i += sprintf(buf, "performance powersave");
577 list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
578 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
579 - (CPUFREQ_NAME_LEN + 2)))
581 i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
584 i += sprintf(&buf[i], "\n");
588 static ssize_t show_cpus(const struct cpumask *mask, char *buf)
593 for_each_cpu(cpu, mask) {
595 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
596 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
597 if (i >= (PAGE_SIZE - 5))
600 i += sprintf(&buf[i], "\n");
605 * show_related_cpus - show the CPUs affected by each transition even if
606 * hw coordination is in use
608 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
610 return show_cpus(policy->related_cpus, buf);
614 * show_affected_cpus - show the CPUs affected by each transition
616 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
618 return show_cpus(policy->cpus, buf);
621 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
622 const char *buf, size_t count)
624 unsigned int freq = 0;
627 if (!policy->governor || !policy->governor->store_setspeed)
630 ret = sscanf(buf, "%u", &freq);
634 policy->governor->store_setspeed(policy, freq);
639 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
641 if (!policy->governor || !policy->governor->show_setspeed)
642 return sprintf(buf, "<unsupported>\n");
644 return policy->governor->show_setspeed(policy, buf);
648 * show_bios_limit - show the current cpufreq HW/BIOS limitation
650 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
654 if (cpufreq_driver->bios_limit) {
655 ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
657 return sprintf(buf, "%u\n", limit);
659 return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
662 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
663 cpufreq_freq_attr_ro(cpuinfo_min_freq);
664 cpufreq_freq_attr_ro(cpuinfo_max_freq);
665 cpufreq_freq_attr_ro(cpuinfo_transition_latency);
666 cpufreq_freq_attr_ro(scaling_available_governors);
667 cpufreq_freq_attr_ro(scaling_driver);
668 cpufreq_freq_attr_ro(scaling_cur_freq);
669 cpufreq_freq_attr_ro(bios_limit);
670 cpufreq_freq_attr_ro(related_cpus);
671 cpufreq_freq_attr_ro(affected_cpus);
672 cpufreq_freq_attr_rw(scaling_min_freq);
673 cpufreq_freq_attr_rw(scaling_max_freq);
674 cpufreq_freq_attr_rw(scaling_governor);
675 cpufreq_freq_attr_rw(scaling_setspeed);
677 static struct attribute *default_attrs[] = {
678 &cpuinfo_min_freq.attr,
679 &cpuinfo_max_freq.attr,
680 &cpuinfo_transition_latency.attr,
681 &scaling_min_freq.attr,
682 &scaling_max_freq.attr,
685 &scaling_governor.attr,
686 &scaling_driver.attr,
687 &scaling_available_governors.attr,
688 &scaling_setspeed.attr,
692 struct kobject *cpufreq_global_kobject;
693 EXPORT_SYMBOL(cpufreq_global_kobject);
695 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
696 #define to_attr(a) container_of(a, struct freq_attr, attr)
698 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
700 struct cpufreq_policy *policy = to_policy(kobj);
701 struct freq_attr *fattr = to_attr(attr);
702 ssize_t ret = -EINVAL;
703 policy = cpufreq_cpu_get_sysfs(policy->cpu);
707 if (lock_policy_rwsem_read(policy->cpu) < 0)
711 ret = fattr->show(policy, buf);
715 unlock_policy_rwsem_read(policy->cpu);
717 cpufreq_cpu_put_sysfs(policy);
722 static ssize_t store(struct kobject *kobj, struct attribute *attr,
723 const char *buf, size_t count)
725 struct cpufreq_policy *policy = to_policy(kobj);
726 struct freq_attr *fattr = to_attr(attr);
727 ssize_t ret = -EINVAL;
728 policy = cpufreq_cpu_get_sysfs(policy->cpu);
732 if (lock_policy_rwsem_write(policy->cpu) < 0)
736 ret = fattr->store(policy, buf, count);
740 unlock_policy_rwsem_write(policy->cpu);
742 cpufreq_cpu_put_sysfs(policy);
747 static void cpufreq_sysfs_release(struct kobject *kobj)
749 struct cpufreq_policy *policy = to_policy(kobj);
750 pr_debug("last reference is dropped\n");
751 complete(&policy->kobj_unregister);
754 static const struct sysfs_ops sysfs_ops = {
759 static struct kobj_type ktype_cpufreq = {
760 .sysfs_ops = &sysfs_ops,
761 .default_attrs = default_attrs,
762 .release = cpufreq_sysfs_release,
765 /* symlink affected CPUs */
766 static int cpufreq_add_dev_symlink(unsigned int cpu,
767 struct cpufreq_policy *policy)
772 for_each_cpu(j, policy->cpus) {
773 struct cpufreq_policy *managed_policy;
774 struct device *cpu_dev;
779 pr_debug("CPU %u already managed, adding link\n", j);
780 managed_policy = cpufreq_cpu_get(cpu);
781 cpu_dev = get_cpu_device(j);
782 ret = sysfs_create_link(&cpu_dev->kobj, &policy->kobj,
785 cpufreq_cpu_put(managed_policy);
792 static int cpufreq_add_dev_interface(unsigned int cpu,
793 struct cpufreq_policy *policy,
796 struct cpufreq_policy new_policy;
797 struct freq_attr **drv_attr;
802 /* prepare interface data */
803 ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
804 &dev->kobj, "cpufreq");
808 /* set up files for this cpu device */
809 drv_attr = cpufreq_driver->attr;
810 while ((drv_attr) && (*drv_attr)) {
811 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
813 goto err_out_kobj_put;
816 if (cpufreq_driver->get) {
817 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
819 goto err_out_kobj_put;
821 if (cpufreq_driver->target) {
822 ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
824 goto err_out_kobj_put;
826 if (cpufreq_driver->bios_limit) {
827 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
829 goto err_out_kobj_put;
832 write_lock_irqsave(&cpufreq_driver_lock, flags);
833 for_each_cpu(j, policy->cpus) {
834 per_cpu(cpufreq_cpu_data, j) = policy;
835 per_cpu(cpufreq_policy_cpu, j) = policy->cpu;
837 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
839 ret = cpufreq_add_dev_symlink(cpu, policy);
841 goto err_out_kobj_put;
843 memcpy(&new_policy, policy, sizeof(struct cpufreq_policy));
844 /* assure that the starting sequence is run in __cpufreq_set_policy */
845 policy->governor = NULL;
847 /* set default policy */
848 ret = __cpufreq_set_policy(policy, &new_policy);
849 policy->user_policy.policy = policy->policy;
850 policy->user_policy.governor = policy->governor;
853 pr_debug("setting policy failed\n");
854 if (cpufreq_driver->exit)
855 cpufreq_driver->exit(policy);
860 kobject_put(&policy->kobj);
861 wait_for_completion(&policy->kobj_unregister);
865 #ifdef CONFIG_HOTPLUG_CPU
866 static int cpufreq_add_policy_cpu(unsigned int cpu, unsigned int sibling,
869 struct cpufreq_policy *policy;
870 int ret = 0, has_target = !!cpufreq_driver->target;
873 policy = cpufreq_cpu_get(sibling);
877 __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
879 lock_policy_rwsem_write(sibling);
881 write_lock_irqsave(&cpufreq_driver_lock, flags);
883 cpumask_set_cpu(cpu, policy->cpus);
884 per_cpu(cpufreq_policy_cpu, cpu) = policy->cpu;
885 per_cpu(cpufreq_cpu_data, cpu) = policy;
886 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
888 unlock_policy_rwsem_write(sibling);
891 __cpufreq_governor(policy, CPUFREQ_GOV_START);
892 __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
895 ret = sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq");
897 cpufreq_cpu_put(policy);
906 * cpufreq_add_dev - add a CPU device
908 * Adds the cpufreq interface for a CPU device.
910 * The Oracle says: try running cpufreq registration/unregistration concurrently
911 * with with cpu hotplugging and all hell will break loose. Tried to clean this
912 * mess up, but more thorough testing is needed. - Mathieu
914 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
916 unsigned int j, cpu = dev->id;
918 struct cpufreq_policy *policy;
920 #ifdef CONFIG_HOTPLUG_CPU
921 struct cpufreq_governor *gov;
925 if (cpu_is_offline(cpu))
928 pr_debug("adding CPU %u\n", cpu);
931 /* check whether a different CPU already registered this
932 * CPU because it is in the same boat. */
933 policy = cpufreq_cpu_get(cpu);
934 if (unlikely(policy)) {
935 cpufreq_cpu_put(policy);
939 #ifdef CONFIG_HOTPLUG_CPU
940 /* Check if this cpu was hot-unplugged earlier and has siblings */
941 read_lock_irqsave(&cpufreq_driver_lock, flags);
942 for_each_online_cpu(sibling) {
943 struct cpufreq_policy *cp = per_cpu(cpufreq_cpu_data, sibling);
944 if (cp && cpumask_test_cpu(cpu, cp->related_cpus)) {
945 read_unlock_irqrestore(&cpufreq_driver_lock, flags);
946 return cpufreq_add_policy_cpu(cpu, sibling, dev);
949 read_unlock_irqrestore(&cpufreq_driver_lock, flags);
953 if (!try_module_get(cpufreq_driver->owner)) {
958 policy = kzalloc(sizeof(struct cpufreq_policy), GFP_KERNEL);
962 if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
963 goto err_free_policy;
965 if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
966 goto err_free_cpumask;
969 policy->governor = CPUFREQ_DEFAULT_GOVERNOR;
970 cpumask_copy(policy->cpus, cpumask_of(cpu));
972 /* Initially set CPU itself as the policy_cpu */
973 per_cpu(cpufreq_policy_cpu, cpu) = cpu;
975 init_completion(&policy->kobj_unregister);
976 INIT_WORK(&policy->update, handle_update);
978 /* call driver. From then on the cpufreq must be able
979 * to accept all calls to ->verify and ->setpolicy for this CPU
981 ret = cpufreq_driver->init(policy);
983 pr_debug("initialization failed\n");
984 goto err_set_policy_cpu;
987 /* related cpus should atleast have policy->cpus */
988 cpumask_or(policy->related_cpus, policy->related_cpus, policy->cpus);
991 * affected cpus must always be the one, which are online. We aren't
992 * managing offline cpus here.
994 cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
996 policy->user_policy.min = policy->min;
997 policy->user_policy.max = policy->max;
999 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1000 CPUFREQ_START, policy);
1002 #ifdef CONFIG_HOTPLUG_CPU
1003 gov = __find_governor(per_cpu(cpufreq_cpu_governor, cpu));
1005 policy->governor = gov;
1006 pr_debug("Restoring governor %s for cpu %d\n",
1007 policy->governor->name, cpu);
1011 ret = cpufreq_add_dev_interface(cpu, policy, dev);
1013 goto err_out_unregister;
1015 kobject_uevent(&policy->kobj, KOBJ_ADD);
1016 module_put(cpufreq_driver->owner);
1017 pr_debug("initialization complete\n");
1022 write_lock_irqsave(&cpufreq_driver_lock, flags);
1023 for_each_cpu(j, policy->cpus)
1024 per_cpu(cpufreq_cpu_data, j) = NULL;
1025 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1027 kobject_put(&policy->kobj);
1028 wait_for_completion(&policy->kobj_unregister);
1031 per_cpu(cpufreq_policy_cpu, cpu) = -1;
1032 free_cpumask_var(policy->related_cpus);
1034 free_cpumask_var(policy->cpus);
1038 module_put(cpufreq_driver->owner);
1043 static void update_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu)
1047 policy->last_cpu = policy->cpu;
1050 for_each_cpu(j, policy->cpus)
1051 per_cpu(cpufreq_policy_cpu, j) = cpu;
1053 #ifdef CONFIG_CPU_FREQ_TABLE
1054 cpufreq_frequency_table_update_policy_cpu(policy);
1056 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1057 CPUFREQ_UPDATE_POLICY_CPU, policy);
1061 * __cpufreq_remove_dev - remove a CPU device
1063 * Removes the cpufreq interface for a CPU device.
1064 * Caller should already have policy_rwsem in write mode for this CPU.
1065 * This routine frees the rwsem before returning.
1067 static int __cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1069 unsigned int cpu = dev->id, ret, cpus;
1070 unsigned long flags;
1071 struct cpufreq_policy *data;
1072 struct kobject *kobj;
1073 struct completion *cmp;
1074 struct device *cpu_dev;
1076 pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
1078 write_lock_irqsave(&cpufreq_driver_lock, flags);
1080 data = per_cpu(cpufreq_cpu_data, cpu);
1081 per_cpu(cpufreq_cpu_data, cpu) = NULL;
1083 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1086 pr_debug("%s: No cpu_data found\n", __func__);
1090 if (cpufreq_driver->target)
1091 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1093 #ifdef CONFIG_HOTPLUG_CPU
1094 if (!cpufreq_driver->setpolicy)
1095 strncpy(per_cpu(cpufreq_cpu_governor, cpu),
1096 data->governor->name, CPUFREQ_NAME_LEN);
1099 WARN_ON(lock_policy_rwsem_write(cpu));
1100 cpus = cpumask_weight(data->cpus);
1103 cpumask_clear_cpu(cpu, data->cpus);
1104 unlock_policy_rwsem_write(cpu);
1106 if (cpu != data->cpu) {
1107 sysfs_remove_link(&dev->kobj, "cpufreq");
1108 } else if (cpus > 1) {
1109 /* first sibling now owns the new sysfs dir */
1110 cpu_dev = get_cpu_device(cpumask_first(data->cpus));
1111 sysfs_remove_link(&cpu_dev->kobj, "cpufreq");
1112 ret = kobject_move(&data->kobj, &cpu_dev->kobj);
1114 pr_err("%s: Failed to move kobj: %d", __func__, ret);
1116 WARN_ON(lock_policy_rwsem_write(cpu));
1117 cpumask_set_cpu(cpu, data->cpus);
1119 write_lock_irqsave(&cpufreq_driver_lock, flags);
1120 per_cpu(cpufreq_cpu_data, cpu) = data;
1121 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1123 unlock_policy_rwsem_write(cpu);
1125 ret = sysfs_create_link(&cpu_dev->kobj, &data->kobj,
1130 WARN_ON(lock_policy_rwsem_write(cpu));
1131 update_policy_cpu(data, cpu_dev->id);
1132 unlock_policy_rwsem_write(cpu);
1133 pr_debug("%s: policy Kobject moved to cpu: %d from: %d\n",
1134 __func__, cpu_dev->id, cpu);
1137 /* If cpu is last user of policy, free policy */
1139 if (cpufreq_driver->target)
1140 __cpufreq_governor(data, CPUFREQ_GOV_POLICY_EXIT);
1142 lock_policy_rwsem_read(cpu);
1144 cmp = &data->kobj_unregister;
1145 unlock_policy_rwsem_read(cpu);
1148 /* we need to make sure that the underlying kobj is actually
1149 * not referenced anymore by anybody before we proceed with
1152 pr_debug("waiting for dropping of refcount\n");
1153 wait_for_completion(cmp);
1154 pr_debug("wait complete\n");
1156 if (cpufreq_driver->exit)
1157 cpufreq_driver->exit(data);
1159 free_cpumask_var(data->related_cpus);
1160 free_cpumask_var(data->cpus);
1163 pr_debug("%s: removing link, cpu: %d\n", __func__, cpu);
1164 cpufreq_cpu_put(data);
1165 if (cpufreq_driver->target) {
1166 __cpufreq_governor(data, CPUFREQ_GOV_START);
1167 __cpufreq_governor(data, CPUFREQ_GOV_LIMITS);
1171 per_cpu(cpufreq_policy_cpu, cpu) = -1;
1176 static int cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1178 unsigned int cpu = dev->id;
1181 if (cpu_is_offline(cpu))
1184 retval = __cpufreq_remove_dev(dev, sif);
1189 static void handle_update(struct work_struct *work)
1191 struct cpufreq_policy *policy =
1192 container_of(work, struct cpufreq_policy, update);
1193 unsigned int cpu = policy->cpu;
1194 pr_debug("handle_update for cpu %u called\n", cpu);
1195 cpufreq_update_policy(cpu);
1199 * cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're in deep trouble.
1201 * @old_freq: CPU frequency the kernel thinks the CPU runs at
1202 * @new_freq: CPU frequency the CPU actually runs at
1204 * We adjust to current frequency first, and need to clean up later.
1205 * So either call to cpufreq_update_policy() or schedule handle_update()).
1207 static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,
1208 unsigned int new_freq)
1210 struct cpufreq_policy *policy;
1211 struct cpufreq_freqs freqs;
1212 unsigned long flags;
1215 pr_debug("Warning: CPU frequency out of sync: cpufreq and timing "
1216 "core thinks of %u, is %u kHz.\n", old_freq, new_freq);
1218 freqs.old = old_freq;
1219 freqs.new = new_freq;
1221 read_lock_irqsave(&cpufreq_driver_lock, flags);
1222 policy = per_cpu(cpufreq_cpu_data, cpu);
1223 read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1225 cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE);
1226 cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE);
1231 * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1234 * This is the last known freq, without actually getting it from the driver.
1235 * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1237 unsigned int cpufreq_quick_get(unsigned int cpu)
1239 struct cpufreq_policy *policy;
1240 unsigned int ret_freq = 0;
1242 if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
1243 return cpufreq_driver->get(cpu);
1245 policy = cpufreq_cpu_get(cpu);
1247 ret_freq = policy->cur;
1248 cpufreq_cpu_put(policy);
1253 EXPORT_SYMBOL(cpufreq_quick_get);
1256 * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1259 * Just return the max possible frequency for a given CPU.
1261 unsigned int cpufreq_quick_get_max(unsigned int cpu)
1263 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1264 unsigned int ret_freq = 0;
1267 ret_freq = policy->max;
1268 cpufreq_cpu_put(policy);
1273 EXPORT_SYMBOL(cpufreq_quick_get_max);
1276 static unsigned int __cpufreq_get(unsigned int cpu)
1278 struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1279 unsigned int ret_freq = 0;
1281 if (!cpufreq_driver->get)
1284 ret_freq = cpufreq_driver->get(cpu);
1286 if (ret_freq && policy->cur &&
1287 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1288 /* verify no discrepancy between actual and
1289 saved value exists */
1290 if (unlikely(ret_freq != policy->cur)) {
1291 cpufreq_out_of_sync(cpu, policy->cur, ret_freq);
1292 schedule_work(&policy->update);
1300 * cpufreq_get - get the current CPU frequency (in kHz)
1303 * Get the CPU current (static) CPU frequency
1305 unsigned int cpufreq_get(unsigned int cpu)
1307 unsigned int ret_freq = 0;
1308 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1313 if (unlikely(lock_policy_rwsem_read(cpu)))
1316 ret_freq = __cpufreq_get(cpu);
1318 unlock_policy_rwsem_read(cpu);
1321 cpufreq_cpu_put(policy);
1325 EXPORT_SYMBOL(cpufreq_get);
1327 static struct subsys_interface cpufreq_interface = {
1329 .subsys = &cpu_subsys,
1330 .add_dev = cpufreq_add_dev,
1331 .remove_dev = cpufreq_remove_dev,
1336 * cpufreq_suspend() - Suspend CPUFreq governors
1338 * Called during system wide Suspend/Hibernate cycles for suspending governors
1339 * as some platforms can't change frequency after this point in suspend cycle.
1340 * Because some of the devices (like: i2c, regulators, etc) they use for
1341 * changing frequency are suspended quickly after this point.
1343 void cpufreq_suspend(void)
1345 struct cpufreq_policy *policy;
1348 if (!cpufreq_driver)
1354 pr_debug("%s: Suspending Governors\n", __func__);
1356 for_each_possible_cpu(cpu) {
1357 if (!cpu_online(cpu))
1360 policy = cpufreq_cpu_get(cpu);
1362 if (__cpufreq_governor(policy, CPUFREQ_GOV_STOP))
1363 pr_err("%s: Failed to stop governor for policy: %p\n",
1365 else if (cpufreq_driver->suspend
1366 && cpufreq_driver->suspend(policy))
1367 pr_err("%s: Failed to suspend driver: %p\n", __func__,
1371 cpufreq_suspended = true;
1375 * cpufreq_resume() - Resume CPUFreq governors
1377 * Called during system wide Suspend/Hibernate cycle for resuming governors that
1378 * are suspended with cpufreq_suspend().
1380 void cpufreq_resume(void)
1382 struct cpufreq_policy *policy;
1385 if (!cpufreq_driver)
1391 pr_debug("%s: Resuming Governors\n", __func__);
1393 cpufreq_suspended = false;
1395 for_each_possible_cpu(cpu) {
1396 if (!cpu_online(cpu))
1399 policy = cpufreq_cpu_get(cpu);
1401 if (__cpufreq_governor(policy, CPUFREQ_GOV_START)
1402 || __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS))
1403 pr_err("%s: Failed to start governor for policy: %p\n",
1405 else if (cpufreq_driver->resume
1406 && cpufreq_driver->resume(policy))
1407 pr_err("%s: Failed to resume driver: %p\n", __func__,
1411 * schedule call cpufreq_update_policy() for boot CPU, i.e. last
1412 * policy in list. It will verify that the current freq is in
1413 * sync with what we believe it to be.
1416 schedule_work(&policy->update);
1421 * cpufreq_get_current_driver - return current driver's name
1423 * Return the name string of the currently loaded cpufreq driver
1426 const char *cpufreq_get_current_driver(void)
1429 return cpufreq_driver->name;
1433 EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
1435 /*********************************************************************
1436 * NOTIFIER LISTS INTERFACE *
1437 *********************************************************************/
1440 * cpufreq_register_notifier - register a driver with cpufreq
1441 * @nb: notifier function to register
1442 * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1444 * Add a driver to one of two lists: either a list of drivers that
1445 * are notified about clock rate changes (once before and once after
1446 * the transition), or a list of drivers that are notified about
1447 * changes in cpufreq policy.
1449 * This function may sleep, and has the same return conditions as
1450 * blocking_notifier_chain_register.
1452 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1456 if (cpufreq_disabled())
1459 WARN_ON(!init_cpufreq_transition_notifier_list_called);
1462 case CPUFREQ_TRANSITION_NOTIFIER:
1463 ret = srcu_notifier_chain_register(
1464 &cpufreq_transition_notifier_list, nb);
1466 case CPUFREQ_POLICY_NOTIFIER:
1467 ret = blocking_notifier_chain_register(
1468 &cpufreq_policy_notifier_list, nb);
1476 EXPORT_SYMBOL(cpufreq_register_notifier);
1480 * cpufreq_unregister_notifier - unregister a driver with cpufreq
1481 * @nb: notifier block to be unregistered
1482 * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1484 * Remove a driver from the CPU frequency notifier list.
1486 * This function may sleep, and has the same return conditions as
1487 * blocking_notifier_chain_unregister.
1489 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1493 if (cpufreq_disabled())
1497 case CPUFREQ_TRANSITION_NOTIFIER:
1498 ret = srcu_notifier_chain_unregister(
1499 &cpufreq_transition_notifier_list, nb);
1501 case CPUFREQ_POLICY_NOTIFIER:
1502 ret = blocking_notifier_chain_unregister(
1503 &cpufreq_policy_notifier_list, nb);
1511 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1514 /*********************************************************************
1516 *********************************************************************/
1519 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1520 unsigned int target_freq,
1521 unsigned int relation)
1523 int retval = -EINVAL;
1524 unsigned int old_target_freq = target_freq;
1526 if (cpufreq_disabled())
1529 /* Make sure that target_freq is within supported range */
1530 if (target_freq > policy->max)
1531 target_freq = policy->max;
1532 if (target_freq < policy->min)
1533 target_freq = policy->min;
1535 pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
1536 policy->cpu, target_freq, relation, old_target_freq);
1538 if (target_freq == policy->cur)
1541 if (cpufreq_driver->target)
1542 retval = cpufreq_driver->target(policy, target_freq, relation);
1546 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1548 int cpufreq_driver_target(struct cpufreq_policy *policy,
1549 unsigned int target_freq,
1550 unsigned int relation)
1554 policy = cpufreq_cpu_get(policy->cpu);
1558 if (unlikely(lock_policy_rwsem_write(policy->cpu)))
1561 ret = __cpufreq_driver_target(policy, target_freq, relation);
1563 unlock_policy_rwsem_write(policy->cpu);
1566 cpufreq_cpu_put(policy);
1570 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
1572 int __cpufreq_driver_getavg(struct cpufreq_policy *policy, unsigned int cpu)
1576 if (cpufreq_disabled())
1579 if (!cpufreq_driver->getavg)
1582 policy = cpufreq_cpu_get(policy->cpu);
1586 ret = cpufreq_driver->getavg(policy, cpu);
1588 cpufreq_cpu_put(policy);
1591 EXPORT_SYMBOL_GPL(__cpufreq_driver_getavg);
1594 * when "event" is CPUFREQ_GOV_LIMITS
1597 static int __cpufreq_governor(struct cpufreq_policy *policy,
1602 /* Only must be defined when default governor is known to have latency
1603 restrictions, like e.g. conservative or ondemand.
1604 That this is the case is already ensured in Kconfig
1606 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
1607 struct cpufreq_governor *gov = &cpufreq_gov_performance;
1609 struct cpufreq_governor *gov = NULL;
1612 /* Don't start any governor operations if we are entering suspend */
1613 if (cpufreq_suspended)
1616 if (policy->governor->max_transition_latency &&
1617 policy->cpuinfo.transition_latency >
1618 policy->governor->max_transition_latency) {
1622 printk(KERN_WARNING "%s governor failed, too long"
1623 " transition latency of HW, fallback"
1624 " to %s governor\n",
1625 policy->governor->name,
1627 policy->governor = gov;
1631 if (!try_module_get(policy->governor->owner))
1634 pr_debug("__cpufreq_governor for CPU %u, event %u\n",
1635 policy->cpu, event);
1637 mutex_lock(&cpufreq_governor_lock);
1638 if ((!policy->governor_enabled && (event == CPUFREQ_GOV_STOP)) ||
1639 (policy->governor_enabled && (event == CPUFREQ_GOV_START))) {
1640 mutex_unlock(&cpufreq_governor_lock);
1644 if (event == CPUFREQ_GOV_STOP)
1645 policy->governor_enabled = false;
1646 else if (event == CPUFREQ_GOV_START)
1647 policy->governor_enabled = true;
1649 mutex_unlock(&cpufreq_governor_lock);
1651 ret = policy->governor->governor(policy, event);
1654 if (event == CPUFREQ_GOV_POLICY_INIT)
1655 policy->governor->initialized++;
1656 else if (event == CPUFREQ_GOV_POLICY_EXIT)
1657 policy->governor->initialized--;
1659 /* Restore original values */
1660 mutex_lock(&cpufreq_governor_lock);
1661 if (event == CPUFREQ_GOV_STOP)
1662 policy->governor_enabled = true;
1663 else if (event == CPUFREQ_GOV_START)
1664 policy->governor_enabled = false;
1665 mutex_unlock(&cpufreq_governor_lock);
1668 /* we keep one module reference alive for
1669 each CPU governed by this CPU */
1670 if ((event != CPUFREQ_GOV_START) || ret)
1671 module_put(policy->governor->owner);
1672 if ((event == CPUFREQ_GOV_STOP) && !ret)
1673 module_put(policy->governor->owner);
1679 int cpufreq_register_governor(struct cpufreq_governor *governor)
1686 if (cpufreq_disabled())
1689 mutex_lock(&cpufreq_governor_mutex);
1691 governor->initialized = 0;
1693 if (__find_governor(governor->name) == NULL) {
1695 list_add(&governor->governor_list, &cpufreq_governor_list);
1698 mutex_unlock(&cpufreq_governor_mutex);
1701 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
1704 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
1706 #ifdef CONFIG_HOTPLUG_CPU
1713 if (cpufreq_disabled())
1716 #ifdef CONFIG_HOTPLUG_CPU
1717 for_each_present_cpu(cpu) {
1718 if (cpu_online(cpu))
1720 if (!strcmp(per_cpu(cpufreq_cpu_governor, cpu), governor->name))
1721 strcpy(per_cpu(cpufreq_cpu_governor, cpu), "\0");
1725 mutex_lock(&cpufreq_governor_mutex);
1726 list_del(&governor->governor_list);
1727 mutex_unlock(&cpufreq_governor_mutex);
1730 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
1734 /*********************************************************************
1735 * POLICY INTERFACE *
1736 *********************************************************************/
1739 * cpufreq_get_policy - get the current cpufreq_policy
1740 * @policy: struct cpufreq_policy into which the current cpufreq_policy
1743 * Reads the current cpufreq policy.
1745 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
1747 struct cpufreq_policy *cpu_policy;
1751 cpu_policy = cpufreq_cpu_get(cpu);
1755 memcpy(policy, cpu_policy, sizeof(struct cpufreq_policy));
1757 cpufreq_cpu_put(cpu_policy);
1760 EXPORT_SYMBOL(cpufreq_get_policy);
1764 * data : current policy.
1765 * policy : policy to be set.
1767 static int __cpufreq_set_policy(struct cpufreq_policy *data,
1768 struct cpufreq_policy *policy)
1770 int ret = 0, failed = 1;
1772 pr_debug("setting new policy for CPU %u: %u - %u kHz\n", policy->cpu,
1773 policy->min, policy->max);
1775 memcpy(&policy->cpuinfo, &data->cpuinfo,
1776 sizeof(struct cpufreq_cpuinfo));
1778 if (policy->min > data->max || policy->max < data->min) {
1783 /* verify the cpu speed can be set within this limit */
1784 ret = cpufreq_driver->verify(policy);
1788 /* adjust if necessary - all reasons */
1789 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1790 CPUFREQ_ADJUST, policy);
1792 /* adjust if necessary - hardware incompatibility*/
1793 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1794 CPUFREQ_INCOMPATIBLE, policy);
1796 /* verify the cpu speed can be set within this limit,
1797 which might be different to the first one */
1798 ret = cpufreq_driver->verify(policy);
1802 /* notification of the new policy */
1803 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1804 CPUFREQ_NOTIFY, policy);
1806 data->min = policy->min;
1807 data->max = policy->max;
1809 pr_debug("new min and max freqs are %u - %u kHz\n",
1810 data->min, data->max);
1812 if (cpufreq_driver->setpolicy) {
1813 data->policy = policy->policy;
1814 pr_debug("setting range\n");
1815 ret = cpufreq_driver->setpolicy(policy);
1817 if (policy->governor != data->governor) {
1818 /* save old, working values */
1819 struct cpufreq_governor *old_gov = data->governor;
1821 pr_debug("governor switch\n");
1823 /* end old governor */
1824 if (data->governor) {
1825 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1826 unlock_policy_rwsem_write(policy->cpu);
1827 __cpufreq_governor(data,
1828 CPUFREQ_GOV_POLICY_EXIT);
1829 lock_policy_rwsem_write(policy->cpu);
1832 /* start new governor */
1833 data->governor = policy->governor;
1834 if (!__cpufreq_governor(data, CPUFREQ_GOV_POLICY_INIT)) {
1835 if (!__cpufreq_governor(data, CPUFREQ_GOV_START)) {
1838 unlock_policy_rwsem_write(policy->cpu);
1839 __cpufreq_governor(data,
1840 CPUFREQ_GOV_POLICY_EXIT);
1841 lock_policy_rwsem_write(policy->cpu);
1846 /* new governor failed, so re-start old one */
1847 pr_debug("starting governor %s failed\n",
1848 data->governor->name);
1850 data->governor = old_gov;
1851 __cpufreq_governor(data,
1852 CPUFREQ_GOV_POLICY_INIT);
1853 __cpufreq_governor(data,
1859 /* might be a policy change, too, so fall through */
1861 pr_debug("governor: change or update limits\n");
1862 __cpufreq_governor(data, CPUFREQ_GOV_LIMITS);
1870 * cpufreq_update_policy - re-evaluate an existing cpufreq policy
1871 * @cpu: CPU which shall be re-evaluated
1873 * Useful for policy notifiers which have different necessities
1874 * at different times.
1876 int cpufreq_update_policy(unsigned int cpu)
1878 struct cpufreq_policy *data = cpufreq_cpu_get(cpu);
1879 struct cpufreq_policy policy;
1887 if (unlikely(lock_policy_rwsem_write(cpu))) {
1892 pr_debug("updating policy for CPU %u\n", cpu);
1893 memcpy(&policy, data, sizeof(struct cpufreq_policy));
1894 policy.min = data->user_policy.min;
1895 policy.max = data->user_policy.max;
1896 policy.policy = data->user_policy.policy;
1897 policy.governor = data->user_policy.governor;
1899 /* BIOS might change freq behind our back
1900 -> ask driver for current freq and notify governors about a change */
1901 if (cpufreq_driver->get) {
1902 policy.cur = cpufreq_driver->get(cpu);
1904 pr_debug("Driver did not initialize current freq");
1905 data->cur = policy.cur;
1907 if (data->cur != policy.cur && cpufreq_driver->target)
1908 cpufreq_out_of_sync(cpu, data->cur,
1913 ret = __cpufreq_set_policy(data, &policy);
1915 unlock_policy_rwsem_write(cpu);
1918 cpufreq_cpu_put(data);
1922 EXPORT_SYMBOL(cpufreq_update_policy);
1924 static int __cpuinit cpufreq_cpu_callback(struct notifier_block *nfb,
1925 unsigned long action, void *hcpu)
1927 unsigned int cpu = (unsigned long)hcpu;
1930 dev = get_cpu_device(cpu);
1934 case CPU_ONLINE_FROZEN:
1935 cpufreq_add_dev(dev, NULL);
1937 case CPU_DOWN_PREPARE:
1938 case CPU_DOWN_PREPARE_FROZEN:
1939 __cpufreq_remove_dev(dev, NULL);
1941 case CPU_DOWN_FAILED:
1942 case CPU_DOWN_FAILED_FROZEN:
1943 cpufreq_add_dev(dev, NULL);
1950 static struct notifier_block __refdata cpufreq_cpu_notifier = {
1951 .notifier_call = cpufreq_cpu_callback,
1954 /*********************************************************************
1955 * REGISTER / UNREGISTER CPUFREQ DRIVER *
1956 *********************************************************************/
1959 * cpufreq_register_driver - register a CPU Frequency driver
1960 * @driver_data: A struct cpufreq_driver containing the values#
1961 * submitted by the CPU Frequency driver.
1963 * Registers a CPU Frequency driver to this core code. This code
1964 * returns zero on success, -EBUSY when another driver got here first
1965 * (and isn't unregistered in the meantime).
1968 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
1970 unsigned long flags;
1973 if (cpufreq_disabled())
1976 if (!driver_data || !driver_data->verify || !driver_data->init ||
1977 ((!driver_data->setpolicy) && (!driver_data->target)))
1980 pr_debug("trying to register driver %s\n", driver_data->name);
1982 if (driver_data->setpolicy)
1983 driver_data->flags |= CPUFREQ_CONST_LOOPS;
1985 write_lock_irqsave(&cpufreq_driver_lock, flags);
1986 if (cpufreq_driver) {
1987 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1990 cpufreq_driver = driver_data;
1991 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1993 ret = subsys_interface_register(&cpufreq_interface);
1995 goto err_null_driver;
1997 if (!(cpufreq_driver->flags & CPUFREQ_STICKY)) {
2001 /* check for at least one working CPU */
2002 for (i = 0; i < nr_cpu_ids; i++)
2003 if (cpu_possible(i) && per_cpu(cpufreq_cpu_data, i)) {
2008 /* if all ->init() calls failed, unregister */
2010 pr_debug("no CPU initialized for driver %s\n",
2016 register_hotcpu_notifier(&cpufreq_cpu_notifier);
2017 pr_debug("driver %s up and running\n", driver_data->name);
2021 subsys_interface_unregister(&cpufreq_interface);
2023 write_lock_irqsave(&cpufreq_driver_lock, flags);
2024 cpufreq_driver = NULL;
2025 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2028 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
2032 * cpufreq_unregister_driver - unregister the current CPUFreq driver
2034 * Unregister the current CPUFreq driver. Only call this if you have
2035 * the right to do so, i.e. if you have succeeded in initialising before!
2036 * Returns zero if successful, and -EINVAL if the cpufreq_driver is
2037 * currently not initialised.
2039 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
2041 unsigned long flags;
2043 if (!cpufreq_driver || (driver != cpufreq_driver))
2046 pr_debug("unregistering driver %s\n", driver->name);
2048 subsys_interface_unregister(&cpufreq_interface);
2049 unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
2051 write_lock_irqsave(&cpufreq_driver_lock, flags);
2052 cpufreq_driver = NULL;
2053 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2057 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
2059 static int __init cpufreq_core_init(void)
2063 if (cpufreq_disabled())
2066 for_each_possible_cpu(cpu) {
2067 per_cpu(cpufreq_policy_cpu, cpu) = -1;
2068 init_rwsem(&per_cpu(cpu_policy_rwsem, cpu));
2071 cpufreq_global_kobject = kobject_create_and_add("cpufreq", &cpu_subsys.dev_root->kobj);
2072 BUG_ON(!cpufreq_global_kobject);
2076 core_initcall(cpufreq_core_init);