drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6  *            (C) 2013 Viresh Kumar <viresh.kumar@linaro.org>
   7  *
   8  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   9  *      Added handling for CPU hotplug
  10  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  11  *      Fix handling for CPU hotplug -- affected CPUs
  12  *
  13  * This program is free software; you can redistribute it and/or modify
  14  * it under the terms of the GNU General Public License version 2 as
  15  * published by the Free Software Foundation.
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/cpu.h>
  21 #include <linux/cpufreq.h>
  22 #include <linux/delay.h>
  23 #include <linux/device.h>
  24 #include <linux/init.h>
  25 #include <linux/kernel_stat.h>
  26 #include <linux/module.h>
  27 #include <linux/mutex.h>
  28 #include <linux/slab.h>
  29 #include <linux/syscore_ops.h>
  30 #include <linux/tick.h>
  31 #include <trace/events/power.h>
  32
  33 /**
  34  * The "cpufreq driver" - the arch- or hardware-dependent low
  35  * level driver of CPUFreq support, and its spinlock. This lock
  36  * also protects the cpufreq_cpu_data array.
  37  */
  38 static struct cpufreq_driver *cpufreq_driver;
  39 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
  40 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data_fallback);
  41 static DEFINE_RWLOCK(cpufreq_driver_lock);
  42 static DEFINE_MUTEX(cpufreq_governor_lock);
  43 static LIST_HEAD(cpufreq_policy_list);
  44
  45 #ifdef CONFIG_HOTPLUG_CPU
  46 /* This one keeps track of the previously set governor of a removed CPU */
  47 static DEFINE_PER_CPU(char[CPUFREQ_NAME_LEN], cpufreq_cpu_governor);
  48 #endif
  49
  50 static inline bool has_target(void)
  51 {
  52         return cpufreq_driver->target_index || cpufreq_driver->target;
  53 }
  54
  55 /*
  56  * rwsem to guarantee that cpufreq driver module doesn't unload during critical
  57  * sections
  58  */
  59 static DECLARE_RWSEM(cpufreq_rwsem);
  60
  61 /* internal prototypes */
  62 static int __cpufreq_governor(struct cpufreq_policy *policy,
  63                 unsigned int event);
  64 static unsigned int __cpufreq_get(unsigned int cpu);
  65 static void handle_update(struct work_struct *work);
  66
  67 /**
  68  * Two notifier lists: the "policy" list is involved in the
  69  * validation process for a new CPU frequency policy; the
  70  * "transition" list for kernel code that needs to handle
  71  * changes to devices when the CPU clock speed changes.
  72  * The mutex locks both lists.
  73  */
  74 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
  75 static struct srcu_notifier_head cpufreq_transition_notifier_list;
  76
  77 static bool init_cpufreq_transition_notifier_list_called;
  78 static int __init init_cpufreq_transition_notifier_list(void)
  79 {
  80         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
  81         init_cpufreq_transition_notifier_list_called = true;
  82         return 0;
  83 }
  84 pure_initcall(init_cpufreq_transition_notifier_list);
  85
  86 static int off __read_mostly;
  87 static int cpufreq_disabled(void)
  88 {
  89         return off;
  90 }
  91 void disable_cpufreq(void)
  92 {
  93         off = 1;
  94 }
  95 static LIST_HEAD(cpufreq_governor_list);
  96 static DEFINE_MUTEX(cpufreq_governor_mutex);
  97
  98 bool have_governor_per_policy(void)
  99 {
 100         return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY);
 101 }
 102 EXPORT_SYMBOL_GPL(have_governor_per_policy);
 103
 104 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
 105 {
 106         if (have_governor_per_policy())
 107                 return &policy->kobj;
 108         else
 109                 return cpufreq_global_kobject;
 110 }
 111 EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
 112
 113 static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
 114 {
 115         u64 idle_time;
 116         u64 cur_wall_time;
 117         u64 busy_time;
 118
 119         cur_wall_time = jiffies64_to_cputime64(get_jiffies_64());
 120
 121         busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER];
 122         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM];
 123         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ];
 124         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ];
 125         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL];
 126         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE];
 127
 128         idle_time = cur_wall_time - busy_time;
 129         if (wall)
 130                 *wall = cputime_to_usecs(cur_wall_time);
 131
 132         return cputime_to_usecs(idle_time);
 133 }
 134
 135 u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
 136 {
 137         u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);
 138
 139         if (idle_time == -1ULL)
 140                 return get_cpu_idle_time_jiffy(cpu, wall);
 141         else if (!io_busy)
 142                 idle_time += get_cpu_iowait_time_us(cpu, wall);
 143
 144         return idle_time;
 145 }
 146 EXPORT_SYMBOL_GPL(get_cpu_idle_time);
 147
 148 /*
 149  * This is a generic cpufreq init() routine which can be used by cpufreq
 150  * drivers of SMP systems. It will do following:
 151  * - validate & show freq table passed
 152  * - set policies transition latency
 153  * - policy->cpus with all possible CPUs
 154  */
 155 int cpufreq_generic_init(struct cpufreq_policy *policy,
 156                 struct cpufreq_frequency_table *table,
 157                 unsigned int transition_latency)
 158 {
 159         int ret;
 160
 161         ret = cpufreq_table_validate_and_show(policy, table);
 162         if (ret) {
 163                 pr_err("%s: invalid frequency table: %d\n", __func__, ret);
 164                 return ret;
 165         }
 166
 167         policy->cpuinfo.transition_latency = transition_latency;
 168
 169         /*
 170          * The driver only supports the SMP configuartion where all processors
 171          * share the clock and voltage and clock.
 172          */
 173         cpumask_setall(policy->cpus);
 174
 175         return 0;
 176 }
 177 EXPORT_SYMBOL_GPL(cpufreq_generic_init);
 178
 179 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 180 {
 181         struct cpufreq_policy *policy = NULL;
 182         unsigned long flags;
 183
 184         if (cpufreq_disabled() || (cpu >= nr_cpu_ids))
 185                 return NULL;
 186
 187         if (!down_read_trylock(&cpufreq_rwsem))
 188                 return NULL;
 189
 190         /* get the cpufreq driver */
 191         read_lock_irqsave(&cpufreq_driver_lock, flags);
 192
 193         if (cpufreq_driver) {
 194                 /* get the CPU */
 195                 policy = per_cpu(cpufreq_cpu_data, cpu);
 196                 if (policy)
 197                         kobject_get(&policy->kobj);
 198         }
 199
 200         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 201
 202         if (!policy)
 203                 up_read(&cpufreq_rwsem);
 204
 205         return policy;
 206 }
 207 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 208
 209 void cpufreq_cpu_put(struct cpufreq_policy *policy)
 210 {
 211         if (cpufreq_disabled())
 212                 return;
 213
 214         kobject_put(&policy->kobj);
 215         up_read(&cpufreq_rwsem);
 216 }
 217 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 218
 219 /*********************************************************************
 220  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 221  *********************************************************************/
 222
 223 /**
 224  * adjust_jiffies - adjust the system "loops_per_jiffy"
 225  *
 226  * This function alters the system "loops_per_jiffy" for the clock
 227  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 228  * systems as each CPU might be scaled differently. So, use the arch
 229  * per-CPU loops_per_jiffy value wherever possible.
 230  */
 231 #ifndef CONFIG_SMP
 232 static unsigned long l_p_j_ref;
 233 static unsigned int l_p_j_ref_freq;
 234
 235 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 236 {
 237         if (ci->flags & CPUFREQ_CONST_LOOPS)
 238                 return;
 239
 240         if (!l_p_j_ref_freq) {
 241                 l_p_j_ref = loops_per_jiffy;
 242                 l_p_j_ref_freq = ci->old;
 243                 pr_debug("saving %lu as reference value for loops_per_jiffy; "
 244                         "freq is %u kHz\n", l_p_j_ref, l_p_j_ref_freq);
 245         }
 246         if ((val == CPUFREQ_POSTCHANGE && ci->old != ci->new) ||
 247             (val == CPUFREQ_RESUMECHANGE || val == CPUFREQ_SUSPENDCHANGE)) {
 248                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 249                                                                 ci->new);
 250                 pr_debug("scaling loops_per_jiffy to %lu "
 251                         "for frequency %u kHz\n", loops_per_jiffy, ci->new);
 252         }
 253 }
 254 #else
 255 static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 256 {
 257         return;
 258 }
 259 #endif
 260
 261 static void __cpufreq_notify_transition(struct cpufreq_policy *policy,
 262                 struct cpufreq_freqs *freqs, unsigned int state)
 263 {
 264         BUG_ON(irqs_disabled());
 265
 266         if (cpufreq_disabled())
 267                 return;
 268
 269         freqs->flags = cpufreq_driver->flags;
 270         pr_debug("notification %u of frequency transition to %u kHz\n",
 271                 state, freqs->new);
 272
 273         switch (state) {
 274
 275         case CPUFREQ_PRECHANGE:
 276                 /* detect if the driver reported a value as "old frequency"
 277                  * which is not equal to what the cpufreq core thinks is
 278                  * "old frequency".
 279                  */
 280                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 281                         if ((policy) && (policy->cpu == freqs->cpu) &&
 282                             (policy->cur) && (policy->cur != freqs->old)) {
 283                                 pr_debug("Warning: CPU frequency is"
 284                                         " %u, cpufreq assumed %u kHz.\n",
 285                                         freqs->old, policy->cur);
 286                                 freqs->old = policy->cur;
 287                         }
 288                 }
 289                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 290                                 CPUFREQ_PRECHANGE, freqs);
 291                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 292                 break;
 293
 294         case CPUFREQ_POSTCHANGE:
 295                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 296                 pr_debug("FREQ: %lu - CPU: %lu", (unsigned long)freqs->new,
 297                         (unsigned long)freqs->cpu);
 298                 trace_cpu_frequency(freqs->new, freqs->cpu);
 299                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 300                                 CPUFREQ_POSTCHANGE, freqs);
 301                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 302                         policy->cur = freqs->new;
 303                 break;
 304         }
 305 }
 306
 307 /**
 308  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 309  * on frequency transition.
 310  *
 311  * This function calls the transition notifiers and the "adjust_jiffies"
 312  * function. It is called twice on all CPU frequency changes that have
 313  * external effects.
 314  */
 315 void cpufreq_notify_transition(struct cpufreq_policy *policy,
 316                 struct cpufreq_freqs *freqs, unsigned int state)
 317 {
 318         for_each_cpu(freqs->cpu, policy->cpus)
 319                 __cpufreq_notify_transition(policy, freqs, state);
 320 }
 321 EXPORT_SYMBOL_GPL(cpufreq_notify_transition);
 322
 323
 324 /*********************************************************************
 325  *                          SYSFS INTERFACE                          *
 326  *********************************************************************/
 327
 328 static struct cpufreq_governor *__find_governor(const char *str_governor)
 329 {
 330         struct cpufreq_governor *t;
 331
 332         list_for_each_entry(t, &cpufreq_governor_list, governor_list)
 333                 if (!strnicmp(str_governor, t->name, CPUFREQ_NAME_LEN))
 334                         return t;
 335
 336         return NULL;
 337 }
 338
 339 /**
 340  * cpufreq_parse_governor - parse a governor string
 341  */
 342 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
 343                                 struct cpufreq_governor **governor)
 344 {
 345         int err = -EINVAL;
 346
 347         if (!cpufreq_driver)
 348                 goto out;
 349
 350         if (cpufreq_driver->setpolicy) {
 351                 if (!strnicmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 352                         *policy = CPUFREQ_POLICY_PERFORMANCE;
 353                         err = 0;
 354                 } else if (!strnicmp(str_governor, "powersave",
 355                                                 CPUFREQ_NAME_LEN)) {
 356                         *policy = CPUFREQ_POLICY_POWERSAVE;
 357                         err = 0;
 358                 }
 359         } else if (has_target()) {
 360                 struct cpufreq_governor *t;
 361
 362                 mutex_lock(&cpufreq_governor_mutex);
 363
 364                 t = __find_governor(str_governor);
 365
 366                 if (t == NULL) {
 367                         int ret;
 368
 369                         mutex_unlock(&cpufreq_governor_mutex);
 370                         ret = request_module("cpufreq_%s", str_governor);
 371                         mutex_lock(&cpufreq_governor_mutex);
 372
 373                         if (ret == 0)
 374                                 t = __find_governor(str_governor);
 375                 }
 376
 377                 if (t != NULL) {
 378                         *governor = t;
 379                         err = 0;
 380                 }
 381
 382                 mutex_unlock(&cpufreq_governor_mutex);
 383         }
 384 out:
 385         return err;
 386 }
 387
 388 /**
 389  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 390  * print out cpufreq information
 391  *
 392  * Write out information from cpufreq_driver->policy[cpu]; object must be
 393  * "unsigned int".
 394  */
 395
 396 #define show_one(file_name, object)                     \
 397 static ssize_t show_##file_name                         \
 398 (struct cpufreq_policy *policy, char *buf)              \
 399 {                                                       \
 400         return sprintf(buf, "%u\n", policy->object);    \
 401 }
 402
 403 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 404 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 405 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
 406 show_one(scaling_min_freq, min);
 407 show_one(scaling_max_freq, max);
 408 show_one(scaling_cur_freq, cur);
 409
 410 static int cpufreq_set_policy(struct cpufreq_policy *policy,
 411                                 struct cpufreq_policy *new_policy);
 412
 413 /**
 414  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 415  */
 416 #define store_one(file_name, object)                    \
 417 static ssize_t store_##file_name                                        \
 418 (struct cpufreq_policy *policy, const char *buf, size_t count)          \
 419 {                                                                       \
 420         int ret;                                                        \
 421         struct cpufreq_policy new_policy;                               \
 422                                                                         \
 423         ret = cpufreq_get_policy(&new_policy, policy->cpu);             \
 424         if (ret)                                                        \
 425                 return -EINVAL;                                         \
 426                                                                         \
 427         ret = sscanf(buf, "%u", &new_policy.object);                    \
 428         if (ret != 1)                                                   \
 429                 return -EINVAL;                                         \
 430                                                                         \
 431         ret = cpufreq_set_policy(policy, &new_policy);          \
 432         policy->user_policy.object = policy->object;                    \
 433                                                                         \
 434         return ret ? ret : count;                                       \
 435 }
 436
 437 store_one(scaling_min_freq, min);
 438 store_one(scaling_max_freq, max);
 439
 440 /**
 441  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 442  */
 443 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
 444                                         char *buf)
 445 {
 446         unsigned int cur_freq = __cpufreq_get(policy->cpu);
 447         if (!cur_freq)
 448                 return sprintf(buf, "<unknown>");
 449         return sprintf(buf, "%u\n", cur_freq);
 450 }
 451
 452 /**
 453  * show_scaling_governor - show the current policy for the specified CPU
 454  */
 455 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
 456 {
 457         if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
 458                 return sprintf(buf, "powersave\n");
 459         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 460                 return sprintf(buf, "performance\n");
 461         else if (policy->governor)
 462                 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
 463                                 policy->governor->name);
 464         return -EINVAL;
 465 }
 466
 467 /**
 468  * store_scaling_governor - store policy for the specified CPU
 469  */
 470 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
 471                                         const char *buf, size_t count)
 472 {
 473         int ret;
 474         char    str_governor[16];
 475         struct cpufreq_policy new_policy;
 476
 477         ret = cpufreq_get_policy(&new_policy, policy->cpu);
 478         if (ret)
 479                 return ret;
 480
 481         ret = sscanf(buf, "%15s", str_governor);
 482         if (ret != 1)
 483                 return -EINVAL;
 484
 485         if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 486                                                 &new_policy.governor))
 487                 return -EINVAL;
 488
 489         ret = cpufreq_set_policy(policy, &new_policy);
 490
 491         policy->user_policy.policy = policy->policy;
 492         policy->user_policy.governor = policy->governor;
 493
 494         if (ret)
 495                 return ret;
 496         else
 497                 return count;
 498 }
 499
 500 /**
 501  * show_scaling_driver - show the cpufreq driver currently loaded
 502  */
 503 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
 504 {
 505         return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
 506 }
 507
 508 /**
 509  * show_scaling_available_governors - show the available CPUfreq governors
 510  */
 511 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
 512                                                 char *buf)
 513 {
 514         ssize_t i = 0;
 515         struct cpufreq_governor *t;
 516
 517         if (!has_target()) {
 518                 i += sprintf(buf, "performance powersave");
 519                 goto out;
 520         }
 521
 522         list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
 523                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
 524                     - (CPUFREQ_NAME_LEN + 2)))
 525                         goto out;
 526                 i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
 527         }
 528 out:
 529         i += sprintf(&buf[i], "\n");
 530         return i;
 531 }
 532
 533 ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
 534 {
 535         ssize_t i = 0;
 536         unsigned int cpu;
 537
 538         for_each_cpu(cpu, mask) {
 539                 if (i)
 540                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 541                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 542                 if (i >= (PAGE_SIZE - 5))
 543                         break;
 544         }
 545         i += sprintf(&buf[i], "\n");
 546         return i;
 547 }
 548 EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
 549
 550 /**
 551  * show_related_cpus - show the CPUs affected by each transition even if
 552  * hw coordination is in use
 553  */
 554 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
 555 {
 556         return cpufreq_show_cpus(policy->related_cpus, buf);
 557 }
 558
 559 /**
 560  * show_affected_cpus - show the CPUs affected by each transition
 561  */
 562 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
 563 {
 564         return cpufreq_show_cpus(policy->cpus, buf);
 565 }
 566
 567 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 568                                         const char *buf, size_t count)
 569 {
 570         unsigned int freq = 0;
 571         unsigned int ret;
 572
 573         if (!policy->governor || !policy->governor->store_setspeed)
 574                 return -EINVAL;
 575
 576         ret = sscanf(buf, "%u", &freq);
 577         if (ret != 1)
 578                 return -EINVAL;
 579
 580         policy->governor->store_setspeed(policy, freq);
 581
 582         return count;
 583 }
 584
 585 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 586 {
 587         if (!policy->governor || !policy->governor->show_setspeed)
 588                 return sprintf(buf, "<unsupported>\n");
 589
 590         return policy->governor->show_setspeed(policy, buf);
 591 }
 592
 593 /**
 594  * show_bios_limit - show the current cpufreq HW/BIOS limitation
 595  */
 596 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
 597 {
 598         unsigned int limit;
 599         int ret;
 600         if (cpufreq_driver->bios_limit) {
 601                 ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
 602                 if (!ret)
 603                         return sprintf(buf, "%u\n", limit);
 604         }
 605         return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
 606 }
 607
 608 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
 609 cpufreq_freq_attr_ro(cpuinfo_min_freq);
 610 cpufreq_freq_attr_ro(cpuinfo_max_freq);
 611 cpufreq_freq_attr_ro(cpuinfo_transition_latency);
 612 cpufreq_freq_attr_ro(scaling_available_governors);
 613 cpufreq_freq_attr_ro(scaling_driver);
 614 cpufreq_freq_attr_ro(scaling_cur_freq);
 615 cpufreq_freq_attr_ro(bios_limit);
 616 cpufreq_freq_attr_ro(related_cpus);
 617 cpufreq_freq_attr_ro(affected_cpus);
 618 cpufreq_freq_attr_rw(scaling_min_freq);
 619 cpufreq_freq_attr_rw(scaling_max_freq);
 620 cpufreq_freq_attr_rw(scaling_governor);
 621 cpufreq_freq_attr_rw(scaling_setspeed);
 622
 623 static struct attribute *default_attrs[] = {
 624         &cpuinfo_min_freq.attr,
 625         &cpuinfo_max_freq.attr,
 626         &cpuinfo_transition_latency.attr,
 627         &scaling_min_freq.attr,
 628         &scaling_max_freq.attr,
 629         &affected_cpus.attr,
 630         &related_cpus.attr,
 631         &scaling_governor.attr,
 632         &scaling_driver.attr,
 633         &scaling_available_governors.attr,
 634         &scaling_setspeed.attr,
 635         NULL
 636 };
 637
 638 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
 639 #define to_attr(a) container_of(a, struct freq_attr, attr)
 640
 641 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
 642 {
 643         struct cpufreq_policy *policy = to_policy(kobj);
 644         struct freq_attr *fattr = to_attr(attr);
 645         ssize_t ret;
 646
 647         if (!down_read_trylock(&cpufreq_rwsem))
 648                 return -EINVAL;
 649
 650         down_read(&policy->rwsem);
 651
 652         if (fattr->show)
 653                 ret = fattr->show(policy, buf);
 654         else
 655                 ret = -EIO;
 656
 657         up_read(&policy->rwsem);
 658         up_read(&cpufreq_rwsem);
 659
 660         return ret;
 661 }
 662
 663 static ssize_t store(struct kobject *kobj, struct attribute *attr,
 664                      const char *buf, size_t count)
 665 {
 666         struct cpufreq_policy *policy = to_policy(kobj);
 667         struct freq_attr *fattr = to_attr(attr);
 668         ssize_t ret = -EINVAL;
 669
 670         get_online_cpus();
 671
 672         if (!cpu_online(policy->cpu))
 673                 goto unlock;
 674
 675         if (!down_read_trylock(&cpufreq_rwsem))
 676                 goto unlock;
 677
 678         down_write(&policy->rwsem);
 679
 680         if (fattr->store)
 681                 ret = fattr->store(policy, buf, count);
 682         else
 683                 ret = -EIO;
 684
 685         up_write(&policy->rwsem);
 686
 687         up_read(&cpufreq_rwsem);
 688 unlock:
 689         put_online_cpus();
 690
 691         return ret;
 692 }
 693
 694 static void cpufreq_sysfs_release(struct kobject *kobj)
 695 {
 696         struct cpufreq_policy *policy = to_policy(kobj);
 697         pr_debug("last reference is dropped\n");
 698         complete(&policy->kobj_unregister);
 699 }
 700
 701 static const struct sysfs_ops sysfs_ops = {
 702         .show   = show,
 703         .store  = store,
 704 };
 705
 706 static struct kobj_type ktype_cpufreq = {
 707         .sysfs_ops      = &sysfs_ops,
 708         .default_attrs  = default_attrs,
 709         .release        = cpufreq_sysfs_release,
 710 };
 711
 712 struct kobject *cpufreq_global_kobject;
 713 EXPORT_SYMBOL(cpufreq_global_kobject);
 714
 715 static int cpufreq_global_kobject_usage;
 716
 717 int cpufreq_get_global_kobject(void)
 718 {
 719         if (!cpufreq_global_kobject_usage++)
 720                 return kobject_add(cpufreq_global_kobject,
 721                                 &cpu_subsys.dev_root->kobj, "%s", "cpufreq");
 722
 723         return 0;
 724 }
 725 EXPORT_SYMBOL(cpufreq_get_global_kobject);
 726
 727 void cpufreq_put_global_kobject(void)
 728 {
 729         if (!--cpufreq_global_kobject_usage)
 730                 kobject_del(cpufreq_global_kobject);
 731 }
 732 EXPORT_SYMBOL(cpufreq_put_global_kobject);
 733
 734 int cpufreq_sysfs_create_file(const struct attribute *attr)
 735 {
 736         int ret = cpufreq_get_global_kobject();
 737
 738         if (!ret) {
 739                 ret = sysfs_create_file(cpufreq_global_kobject, attr);
 740                 if (ret)
 741                         cpufreq_put_global_kobject();
 742         }
 743
 744         return ret;
 745 }
 746 EXPORT_SYMBOL(cpufreq_sysfs_create_file);
 747
 748 void cpufreq_sysfs_remove_file(const struct attribute *attr)
 749 {
 750         sysfs_remove_file(cpufreq_global_kobject, attr);
 751         cpufreq_put_global_kobject();
 752 }
 753 EXPORT_SYMBOL(cpufreq_sysfs_remove_file);
 754
 755 /* symlink affected CPUs */
 756 static int cpufreq_add_dev_symlink(struct cpufreq_policy *policy)
 757 {
 758         unsigned int j;
 759         int ret = 0;
 760
 761         for_each_cpu(j, policy->cpus) {
 762                 struct device *cpu_dev;
 763
 764                 if (j == policy->cpu)
 765                         continue;
 766
 767                 pr_debug("Adding link for CPU: %u\n", j);
 768                 cpu_dev = get_cpu_device(j);
 769                 ret = sysfs_create_link(&cpu_dev->kobj, &policy->kobj,
 770                                         "cpufreq");
 771                 if (ret)
 772                         break;
 773         }
 774         return ret;
 775 }
 776
 777 static int cpufreq_add_dev_interface(struct cpufreq_policy *policy,
 778                                      struct device *dev)
 779 {
 780         struct freq_attr **drv_attr;
 781         int ret = 0;
 782
 783         /* prepare interface data */
 784         ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
 785                                    &dev->kobj, "cpufreq");
 786         if (ret)
 787                 return ret;
 788
 789         /* set up files for this cpu device */
 790         drv_attr = cpufreq_driver->attr;
 791         while ((drv_attr) && (*drv_attr)) {
 792                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
 793                 if (ret)
 794                         goto err_out_kobj_put;
 795                 drv_attr++;
 796         }
 797         if (cpufreq_driver->get) {
 798                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
 799                 if (ret)
 800                         goto err_out_kobj_put;
 801         }
 802         if (has_target()) {
 803                 ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
 804                 if (ret)
 805                         goto err_out_kobj_put;
 806         }
 807         if (cpufreq_driver->bios_limit) {
 808                 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
 809                 if (ret)
 810                         goto err_out_kobj_put;
 811         }
 812
 813         ret = cpufreq_add_dev_symlink(policy);
 814         if (ret)
 815                 goto err_out_kobj_put;
 816
 817         return ret;
 818
 819 err_out_kobj_put:
 820         kobject_put(&policy->kobj);
 821         wait_for_completion(&policy->kobj_unregister);
 822         return ret;
 823 }
 824
 825 static void cpufreq_init_policy(struct cpufreq_policy *policy)
 826 {
 827         struct cpufreq_policy new_policy;
 828         int ret = 0;
 829
 830         memcpy(&new_policy, policy, sizeof(*policy));
 831
 832         /* Use the default policy if its valid. */
 833         if (cpufreq_driver->setpolicy)
 834                 cpufreq_parse_governor(policy->governor->name,
 835                                         &new_policy.policy, NULL);
 836
 837         /* assure that the starting sequence is run in cpufreq_set_policy */
 838         policy->governor = NULL;
 839
 840         /* set default policy */
 841         ret = cpufreq_set_policy(policy, &new_policy);
 842         policy->user_policy.policy = policy->policy;
 843         policy->user_policy.governor = policy->governor;
 844
 845         if (ret) {
 846                 pr_debug("setting policy failed\n");
 847                 if (cpufreq_driver->exit)
 848                         cpufreq_driver->exit(policy);
 849         }
 850 }
 851
 852 #ifdef CONFIG_HOTPLUG_CPU
 853 static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy,
 854                                   unsigned int cpu, struct device *dev)
 855 {
 856         int ret = 0;
 857         unsigned long flags;
 858
 859         if (has_target()) {
 860                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
 861                 if (ret) {
 862                         pr_err("%s: Failed to stop governor\n", __func__);
 863                         return ret;
 864                 }
 865         }
 866
 867         down_write(&policy->rwsem);
 868
 869         write_lock_irqsave(&cpufreq_driver_lock, flags);
 870
 871         cpumask_set_cpu(cpu, policy->cpus);
 872         per_cpu(cpufreq_cpu_data, cpu) = policy;
 873         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
 874
 875         up_write(&policy->rwsem);
 876
 877         if (has_target()) {
 878                 if ((ret = __cpufreq_governor(policy, CPUFREQ_GOV_START)) ||
 879                         (ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS))) {
 880                         pr_err("%s: Failed to start governor\n", __func__);
 881                         return ret;
 882                 }
 883         }
 884
 885         return sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq");
 886 }
 887 #endif
 888
 889 static struct cpufreq_policy *cpufreq_policy_restore(unsigned int cpu)
 890 {
 891         struct cpufreq_policy *policy;
 892         unsigned long flags;
 893
 894         read_lock_irqsave(&cpufreq_driver_lock, flags);
 895
 896         policy = per_cpu(cpufreq_cpu_data_fallback, cpu);
 897
 898         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 899
 900         return policy;
 901 }
 902
 903 static struct cpufreq_policy *cpufreq_policy_alloc(void)
 904 {
 905         struct cpufreq_policy *policy;
 906
 907         policy = kzalloc(sizeof(*policy), GFP_KERNEL);
 908         if (!policy)
 909                 return NULL;
 910
 911         if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
 912                 goto err_free_policy;
 913
 914         if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
 915                 goto err_free_cpumask;
 916
 917         INIT_LIST_HEAD(&policy->policy_list);
 918         init_rwsem(&policy->rwsem);
 919
 920         return policy;
 921
 922 err_free_cpumask:
 923         free_cpumask_var(policy->cpus);
 924 err_free_policy:
 925         kfree(policy);
 926
 927         return NULL;
 928 }
 929
 930 static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy)
 931 {
 932         struct kobject *kobj;
 933         struct completion *cmp;
 934
 935         down_read(&policy->rwsem);
 936         kobj = &policy->kobj;
 937         cmp = &policy->kobj_unregister;
 938         up_read(&policy->rwsem);
 939         kobject_put(kobj);
 940
 941         /*
 942          * We need to make sure that the underlying kobj is
 943          * actually not referenced anymore by anybody before we
 944          * proceed with unloading.
 945          */
 946         pr_debug("waiting for dropping of refcount\n");
 947         wait_for_completion(cmp);
 948         pr_debug("wait complete\n");
 949 }
 950
 951 static void cpufreq_policy_free(struct cpufreq_policy *policy)
 952 {
 953         free_cpumask_var(policy->related_cpus);
 954         free_cpumask_var(policy->cpus);
 955         kfree(policy);
 956 }
 957
 958 static void update_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu)
 959 {
 960         if (WARN_ON(cpu == policy->cpu))
 961                 return;
 962
 963         down_write(&policy->rwsem);
 964
 965         policy->last_cpu = policy->cpu;
 966         policy->cpu = cpu;
 967
 968         up_write(&policy->rwsem);
 969
 970         cpufreq_frequency_table_update_policy_cpu(policy);
 971         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
 972                         CPUFREQ_UPDATE_POLICY_CPU, policy);
 973 }
 974
 975 static int __cpufreq_add_dev(struct device *dev, struct subsys_interface *sif,
 976                              bool frozen)
 977 {
 978         unsigned int j, cpu = dev->id;
 979         int ret = -ENOMEM;
 980         struct cpufreq_policy *policy;
 981         unsigned long flags;
 982 #ifdef CONFIG_HOTPLUG_CPU
 983         struct cpufreq_policy *tpolicy;
 984         struct cpufreq_governor *gov;
 985 #endif
 986
 987         if (cpu_is_offline(cpu))
 988                 return 0;
 989
 990         pr_debug("adding CPU %u\n", cpu);
 991
 992 #ifdef CONFIG_SMP
 993         /* check whether a different CPU already registered this
 994          * CPU because it is in the same boat. */
 995         policy = cpufreq_cpu_get(cpu);
 996         if (unlikely(policy)) {
 997                 cpufreq_cpu_put(policy);
 998                 return 0;
 999         }
1000 #endif
1001
1002         if (!down_read_trylock(&cpufreq_rwsem))
1003                 return 0;
1004
1005 #ifdef CONFIG_HOTPLUG_CPU
1006         /* Check if this cpu was hot-unplugged earlier and has siblings */
1007         read_lock_irqsave(&cpufreq_driver_lock, flags);
1008         list_for_each_entry(tpolicy, &cpufreq_policy_list, policy_list) {
1009                 if (cpumask_test_cpu(cpu, tpolicy->related_cpus)) {
1010                         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1011                         ret = cpufreq_add_policy_cpu(tpolicy, cpu, dev);
1012                         up_read(&cpufreq_rwsem);
1013                         return ret;
1014                 }
1015         }
1016         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1017 #endif
1018
1019         /*
1020          * Restore the saved policy when doing light-weight init and fall back
1021          * to the full init if that fails.
1022          */
1023         policy = frozen ? cpufreq_policy_restore(cpu) : NULL;
1024         if (!policy) {
1025                 frozen = false;
1026                 policy = cpufreq_policy_alloc();
1027                 if (!policy)
1028                         goto nomem_out;
1029         }
1030
1031         /*
1032          * In the resume path, since we restore a saved policy, the assignment
1033          * to policy->cpu is like an update of the existing policy, rather than
1034          * the creation of a brand new one. So we need to perform this update
1035          * by invoking update_policy_cpu().
1036          */
1037         if (frozen && cpu != policy->cpu)
1038                 update_policy_cpu(policy, cpu);
1039         else
1040                 policy->cpu = cpu;
1041
1042         policy->governor = CPUFREQ_DEFAULT_GOVERNOR;
1043         cpumask_copy(policy->cpus, cpumask_of(cpu));
1044
1045         init_completion(&policy->kobj_unregister);
1046         INIT_WORK(&policy->update, handle_update);
1047
1048         /* call driver. From then on the cpufreq must be able
1049          * to accept all calls to ->verify and ->setpolicy for this CPU
1050          */
1051         ret = cpufreq_driver->init(policy);
1052         if (ret) {
1053                 pr_debug("initialization failed\n");
1054                 goto err_set_policy_cpu;
1055         }
1056
1057         if (cpufreq_driver->get) {
1058                 policy->cur = cpufreq_driver->get(policy->cpu);
1059                 if (!policy->cur) {
1060                         pr_err("%s: ->get() failed\n", __func__);
1061                         goto err_get_freq;
1062                 }
1063         }
1064
1065         /* related cpus should atleast have policy->cpus */
1066         cpumask_or(policy->related_cpus, policy->related_cpus, policy->cpus);
1067
1068         /*
1069          * affected cpus must always be the one, which are online. We aren't
1070          * managing offline cpus here.
1071          */
1072         cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
1073
1074         policy->user_policy.min = policy->min;
1075         policy->user_policy.max = policy->max;
1076
1077         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1078                                      CPUFREQ_START, policy);
1079
1080 #ifdef CONFIG_HOTPLUG_CPU
1081         gov = __find_governor(per_cpu(cpufreq_cpu_governor, cpu));
1082         if (gov) {
1083                 policy->governor = gov;
1084                 pr_debug("Restoring governor %s for cpu %d\n",
1085                        policy->governor->name, cpu);
1086         }
1087 #endif
1088
1089         write_lock_irqsave(&cpufreq_driver_lock, flags);
1090         for_each_cpu(j, policy->cpus)
1091                 per_cpu(cpufreq_cpu_data, j) = policy;
1092         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1093
1094         if (!frozen) {
1095                 ret = cpufreq_add_dev_interface(policy, dev);
1096                 if (ret)
1097                         goto err_out_unregister;
1098         }
1099
1100         write_lock_irqsave(&cpufreq_driver_lock, flags);
1101         list_add(&policy->policy_list, &cpufreq_policy_list);
1102         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1103
1104         cpufreq_init_policy(policy);
1105
1106         kobject_uevent(&policy->kobj, KOBJ_ADD);
1107         up_read(&cpufreq_rwsem);
1108
1109         pr_debug("initialization complete\n");
1110
1111         return 0;
1112
1113 err_out_unregister:
1114         write_lock_irqsave(&cpufreq_driver_lock, flags);
1115         for_each_cpu(j, policy->cpus)
1116                 per_cpu(cpufreq_cpu_data, j) = NULL;
1117         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1118
1119 err_get_freq:
1120         if (cpufreq_driver->exit)
1121                 cpufreq_driver->exit(policy);
1122 err_set_policy_cpu:
1123         if (frozen) {
1124                 /* Do not leave stale fallback data behind. */
1125                 per_cpu(cpufreq_cpu_data_fallback, cpu) = NULL;
1126                 cpufreq_policy_put_kobj(policy);
1127         }
1128         cpufreq_policy_free(policy);
1129
1130 nomem_out:
1131         up_read(&cpufreq_rwsem);
1132
1133         return ret;
1134 }
1135
1136 /**
1137  * cpufreq_add_dev - add a CPU device
1138  *
1139  * Adds the cpufreq interface for a CPU device.
1140  *
1141  * The Oracle says: try running cpufreq registration/unregistration concurrently
1142  * with with cpu hotplugging and all hell will break loose. Tried to clean this
1143  * mess up, but more thorough testing is needed. - Mathieu
1144  */
1145 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1146 {
1147         return __cpufreq_add_dev(dev, sif, false);
1148 }
1149
1150 static int cpufreq_nominate_new_policy_cpu(struct cpufreq_policy *policy,
1151                                            unsigned int old_cpu)
1152 {
1153         struct device *cpu_dev;
1154         int ret;
1155
1156         /* first sibling now owns the new sysfs dir */
1157         cpu_dev = get_cpu_device(cpumask_any_but(policy->cpus, old_cpu));
1158
1159         sysfs_remove_link(&cpu_dev->kobj, "cpufreq");
1160         ret = kobject_move(&policy->kobj, &cpu_dev->kobj);
1161         if (ret) {
1162                 pr_err("%s: Failed to move kobj: %d", __func__, ret);
1163
1164                 down_write(&policy->rwsem);
1165                 cpumask_set_cpu(old_cpu, policy->cpus);
1166                 up_write(&policy->rwsem);
1167
1168                 ret = sysfs_create_link(&cpu_dev->kobj, &policy->kobj,
1169                                         "cpufreq");
1170
1171                 return -EINVAL;
1172         }
1173
1174         return cpu_dev->id;
1175 }
1176
1177 static int __cpufreq_remove_dev_prepare(struct device *dev,
1178                                         struct subsys_interface *sif,
1179                                         bool frozen)
1180 {
1181         unsigned int cpu = dev->id, cpus;
1182         int new_cpu, ret;
1183         unsigned long flags;
1184         struct cpufreq_policy *policy;
1185
1186         pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
1187
1188         write_lock_irqsave(&cpufreq_driver_lock, flags);
1189
1190         policy = per_cpu(cpufreq_cpu_data, cpu);
1191
1192         /* Save the policy somewhere when doing a light-weight tear-down */
1193         if (frozen)
1194                 per_cpu(cpufreq_cpu_data_fallback, cpu) = policy;
1195
1196         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1197
1198         if (!policy) {
1199                 pr_debug("%s: No cpu_data found\n", __func__);
1200                 return -EINVAL;
1201         }
1202
1203         if (has_target()) {
1204                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
1205                 if (ret) {
1206                         pr_err("%s: Failed to stop governor\n", __func__);
1207                         return ret;
1208                 }
1209         }
1210
1211 #ifdef CONFIG_HOTPLUG_CPU
1212         if (!cpufreq_driver->setpolicy)
1213                 strncpy(per_cpu(cpufreq_cpu_governor, cpu),
1214                         policy->governor->name, CPUFREQ_NAME_LEN);
1215 #endif
1216
1217         down_read(&policy->rwsem);
1218         cpus = cpumask_weight(policy->cpus);
1219         up_read(&policy->rwsem);
1220
1221         if (cpu != policy->cpu) {
1222                 if (!frozen)
1223                         sysfs_remove_link(&dev->kobj, "cpufreq");
1224         } else if (cpus > 1) {
1225                 new_cpu = cpufreq_nominate_new_policy_cpu(policy, cpu);
1226                 if (new_cpu >= 0) {
1227                         update_policy_cpu(policy, new_cpu);
1228
1229                         if (!frozen) {
1230                                 pr_debug("%s: policy Kobject moved to cpu: %d from: %d\n",
1231                                                 __func__, new_cpu, cpu);
1232                         }
1233                 }
1234         }
1235
1236         return 0;
1237 }
1238
1239 static int __cpufreq_remove_dev_finish(struct device *dev,
1240                                        struct subsys_interface *sif,
1241                                        bool frozen)
1242 {
1243         unsigned int cpu = dev->id, cpus;
1244         int ret;
1245         unsigned long flags;
1246         struct cpufreq_policy *policy;
1247
1248         read_lock_irqsave(&cpufreq_driver_lock, flags);
1249         policy = per_cpu(cpufreq_cpu_data, cpu);
1250         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1251
1252         if (!policy) {
1253                 pr_debug("%s: No cpu_data found\n", __func__);
1254                 return -EINVAL;
1255         }
1256
1257         down_write(&policy->rwsem);
1258         cpus = cpumask_weight(policy->cpus);
1259
1260         if (cpus > 1)
1261                 cpumask_clear_cpu(cpu, policy->cpus);
1262         up_write(&policy->rwsem);
1263
1264         /* If cpu is last user of policy, free policy */
1265         if (cpus == 1) {
1266                 if (has_target()) {
1267                         ret = __cpufreq_governor(policy,
1268                                         CPUFREQ_GOV_POLICY_EXIT);
1269                         if (ret) {
1270                                 pr_err("%s: Failed to exit governor\n",
1271                                                 __func__);
1272                                 return ret;
1273                         }
1274                 }
1275
1276                 if (!frozen)
1277                         cpufreq_policy_put_kobj(policy);
1278
1279                 /*
1280                  * Perform the ->exit() even during light-weight tear-down,
1281                  * since this is a core component, and is essential for the
1282                  * subsequent light-weight ->init() to succeed.
1283                  */
1284                 if (cpufreq_driver->exit)
1285                         cpufreq_driver->exit(policy);
1286
1287                 /* Remove policy from list of active policies */
1288                 write_lock_irqsave(&cpufreq_driver_lock, flags);
1289                 list_del(&policy->policy_list);
1290                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1291
1292                 if (!frozen)
1293                         cpufreq_policy_free(policy);
1294         } else {
1295                 if (has_target()) {
1296                         if ((ret = __cpufreq_governor(policy, CPUFREQ_GOV_START)) ||
1297                                         (ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS))) {
1298                                 pr_err("%s: Failed to start governor\n",
1299                                                 __func__);
1300                                 return ret;
1301                         }
1302                 }
1303         }
1304
1305         per_cpu(cpufreq_cpu_data, cpu) = NULL;
1306         return 0;
1307 }
1308
1309 /**
1310  * cpufreq_remove_dev - remove a CPU device
1311  *
1312  * Removes the cpufreq interface for a CPU device.
1313  */
1314 static int cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1315 {
1316         unsigned int cpu = dev->id;
1317         int ret;
1318
1319         if (cpu_is_offline(cpu))
1320                 return 0;
1321
1322         ret = __cpufreq_remove_dev_prepare(dev, sif, false);
1323
1324         if (!ret)
1325                 ret = __cpufreq_remove_dev_finish(dev, sif, false);
1326
1327         return ret;
1328 }
1329
1330 static void handle_update(struct work_struct *work)
1331 {
1332         struct cpufreq_policy *policy =
1333                 container_of(work, struct cpufreq_policy, update);
1334         unsigned int cpu = policy->cpu;
1335         pr_debug("handle_update for cpu %u called\n", cpu);
1336         cpufreq_update_policy(cpu);
1337 }
1338
1339 /**
1340  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're
1341  *      in deep trouble.
1342  *      @cpu: cpu number
1343  *      @old_freq: CPU frequency the kernel thinks the CPU runs at
1344  *      @new_freq: CPU frequency the CPU actually runs at
1345  *
1346  *      We adjust to current frequency first, and need to clean up later.
1347  *      So either call to cpufreq_update_policy() or schedule handle_update()).
1348  */
1349 static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,
1350                                 unsigned int new_freq)
1351 {
1352         struct cpufreq_policy *policy;
1353         struct cpufreq_freqs freqs;
1354         unsigned long flags;
1355
1356         pr_debug("Warning: CPU frequency out of sync: cpufreq and timing "
1357                "core thinks of %u, is %u kHz.\n", old_freq, new_freq);
1358
1359         freqs.old = old_freq;
1360         freqs.new = new_freq;
1361
1362         read_lock_irqsave(&cpufreq_driver_lock, flags);
1363         policy = per_cpu(cpufreq_cpu_data, cpu);
1364         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1365
1366         cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE);
1367         cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE);
1368 }
1369
1370 /**
1371  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1372  * @cpu: CPU number
1373  *
1374  * This is the last known freq, without actually getting it from the driver.
1375  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1376  */
1377 unsigned int cpufreq_quick_get(unsigned int cpu)
1378 {
1379         struct cpufreq_policy *policy;
1380         unsigned int ret_freq = 0;
1381
1382         if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
1383                 return cpufreq_driver->get(cpu);
1384
1385         policy = cpufreq_cpu_get(cpu);
1386         if (policy) {
1387                 ret_freq = policy->cur;
1388                 cpufreq_cpu_put(policy);
1389         }
1390
1391         return ret_freq;
1392 }
1393 EXPORT_SYMBOL(cpufreq_quick_get);
1394
1395 /**
1396  * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1397  * @cpu: CPU number
1398  *
1399  * Just return the max possible frequency for a given CPU.
1400  */
1401 unsigned int cpufreq_quick_get_max(unsigned int cpu)
1402 {
1403         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1404         unsigned int ret_freq = 0;
1405
1406         if (policy) {
1407                 ret_freq = policy->max;
1408                 cpufreq_cpu_put(policy);
1409         }
1410
1411         return ret_freq;
1412 }
1413 EXPORT_SYMBOL(cpufreq_quick_get_max);
1414
1415 static unsigned int __cpufreq_get(unsigned int cpu)
1416 {
1417         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1418         unsigned int ret_freq = 0;
1419
1420         if (!cpufreq_driver->get)
1421                 return ret_freq;
1422
1423         ret_freq = cpufreq_driver->get(cpu);
1424
1425         if (ret_freq && policy->cur &&
1426                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1427                 /* verify no discrepancy between actual and
1428                                         saved value exists */
1429                 if (unlikely(ret_freq != policy->cur)) {
1430                         cpufreq_out_of_sync(cpu, policy->cur, ret_freq);
1431                         schedule_work(&policy->update);
1432                 }
1433         }
1434
1435         return ret_freq;
1436 }
1437
1438 /**
1439  * cpufreq_get - get the current CPU frequency (in kHz)
1440  * @cpu: CPU number
1441  *
1442  * Get the CPU current (static) CPU frequency
1443  */
1444 unsigned int cpufreq_get(unsigned int cpu)
1445 {
1446         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1447         unsigned int ret_freq = 0;
1448
1449         if (cpufreq_disabled() || !cpufreq_driver)
1450                 return -ENOENT;
1451
1452         BUG_ON(!policy);
1453
1454         if (!down_read_trylock(&cpufreq_rwsem))
1455                 return 0;
1456
1457         down_read(&policy->rwsem);
1458
1459         ret_freq = __cpufreq_get(cpu);
1460
1461         up_read(&policy->rwsem);
1462         up_read(&cpufreq_rwsem);
1463
1464         return ret_freq;
1465 }
1466 EXPORT_SYMBOL(cpufreq_get);
1467
1468 static struct subsys_interface cpufreq_interface = {
1469         .name           = "cpufreq",
1470         .subsys         = &cpu_subsys,
1471         .add_dev        = cpufreq_add_dev,
1472         .remove_dev     = cpufreq_remove_dev,
1473 };
1474
1475 /**
1476  * cpufreq_bp_suspend - Prepare the boot CPU for system suspend.
1477  *
1478  * This function is only executed for the boot processor.  The other CPUs
1479  * have been put offline by means of CPU hotplug.
1480  */
1481 static int cpufreq_bp_suspend(void)
1482 {
1483         int ret = 0;
1484
1485         int cpu = smp_processor_id();
1486         struct cpufreq_policy *policy;
1487
1488         pr_debug("suspending cpu %u\n", cpu);
1489
1490         /* If there's no policy for the boot CPU, we have nothing to do. */
1491         policy = cpufreq_cpu_get(cpu);
1492         if (!policy)
1493                 return 0;
1494
1495         if (cpufreq_driver->suspend) {
1496                 ret = cpufreq_driver->suspend(policy);
1497                 if (ret)
1498                         printk(KERN_ERR "cpufreq: suspend failed in ->suspend "
1499                                         "step on CPU %u\n", policy->cpu);
1500         }
1501
1502         cpufreq_cpu_put(policy);
1503         return ret;
1504 }
1505
1506 /**
1507  * cpufreq_bp_resume - Restore proper frequency handling of the boot CPU.
1508  *
1509  *      1.) resume CPUfreq hardware support (cpufreq_driver->resume())
1510  *      2.) schedule call cpufreq_update_policy() ASAP as interrupts are
1511  *          restored. It will verify that the current freq is in sync with
1512  *          what we believe it to be. This is a bit later than when it
1513  *          should be, but nonethteless it's better than calling
1514  *          cpufreq_driver->get() here which might re-enable interrupts...
1515  *
1516  * This function is only executed for the boot CPU.  The other CPUs have not
1517  * been turned on yet.
1518  */
1519 static void cpufreq_bp_resume(void)
1520 {
1521         int ret = 0;
1522
1523         int cpu = smp_processor_id();
1524         struct cpufreq_policy *policy;
1525
1526         pr_debug("resuming cpu %u\n", cpu);
1527
1528         /* If there's no policy for the boot CPU, we have nothing to do. */
1529         policy = cpufreq_cpu_get(cpu);
1530         if (!policy)
1531                 return;
1532
1533         if (cpufreq_driver->resume) {
1534                 ret = cpufreq_driver->resume(policy);
1535                 if (ret) {
1536                         printk(KERN_ERR "cpufreq: resume failed in ->resume "
1537                                         "step on CPU %u\n", policy->cpu);
1538                         goto fail;
1539                 }
1540         }
1541
1542         schedule_work(&policy->update);
1543
1544 fail:
1545         cpufreq_cpu_put(policy);
1546 }
1547
1548 static struct syscore_ops cpufreq_syscore_ops = {
1549         .suspend        = cpufreq_bp_suspend,
1550         .resume         = cpufreq_bp_resume,
1551 };
1552
1553 /**
1554  *      cpufreq_get_current_driver - return current driver's name
1555  *
1556  *      Return the name string of the currently loaded cpufreq driver
1557  *      or NULL, if none.
1558  */
1559 const char *cpufreq_get_current_driver(void)
1560 {
1561         if (cpufreq_driver)
1562                 return cpufreq_driver->name;
1563
1564         return NULL;
1565 }
1566 EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
1567
1568 /*********************************************************************
1569  *                     NOTIFIER LISTS INTERFACE                      *
1570  *********************************************************************/
1571
1572 /**
1573  *      cpufreq_register_notifier - register a driver with cpufreq
1574  *      @nb: notifier function to register
1575  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1576  *
1577  *      Add a driver to one of two lists: either a list of drivers that
1578  *      are notified about clock rate changes (once before and once after
1579  *      the transition), or a list of drivers that are notified about
1580  *      changes in cpufreq policy.
1581  *
1582  *      This function may sleep, and has the same return conditions as
1583  *      blocking_notifier_chain_register.
1584  */
1585 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1586 {
1587         int ret;
1588
1589         if (cpufreq_disabled())
1590                 return -EINVAL;
1591
1592         WARN_ON(!init_cpufreq_transition_notifier_list_called);
1593
1594         switch (list) {
1595         case CPUFREQ_TRANSITION_NOTIFIER:
1596                 ret = srcu_notifier_chain_register(
1597                                 &cpufreq_transition_notifier_list, nb);
1598                 break;
1599         case CPUFREQ_POLICY_NOTIFIER:
1600                 ret = blocking_notifier_chain_register(
1601                                 &cpufreq_policy_notifier_list, nb);
1602                 break;
1603         default:
1604                 ret = -EINVAL;
1605         }
1606
1607         return ret;
1608 }
1609 EXPORT_SYMBOL(cpufreq_register_notifier);
1610
1611 /**
1612  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1613  *      @nb: notifier block to be unregistered
1614  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1615  *
1616  *      Remove a driver from the CPU frequency notifier list.
1617  *
1618  *      This function may sleep, and has the same return conditions as
1619  *      blocking_notifier_chain_unregister.
1620  */
1621 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1622 {
1623         int ret;
1624
1625         if (cpufreq_disabled())
1626                 return -EINVAL;
1627
1628         switch (list) {
1629         case CPUFREQ_TRANSITION_NOTIFIER:
1630                 ret = srcu_notifier_chain_unregister(
1631                                 &cpufreq_transition_notifier_list, nb);
1632                 break;
1633         case CPUFREQ_POLICY_NOTIFIER:
1634                 ret = blocking_notifier_chain_unregister(
1635                                 &cpufreq_policy_notifier_list, nb);
1636                 break;
1637         default:
1638                 ret = -EINVAL;
1639         }
1640
1641         return ret;
1642 }
1643 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1644
1645
1646 /*********************************************************************
1647  *                              GOVERNORS                            *
1648  *********************************************************************/
1649
1650 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1651                             unsigned int target_freq,
1652                             unsigned int relation)
1653 {
1654         int retval = -EINVAL;
1655         unsigned int old_target_freq = target_freq;
1656
1657         if (cpufreq_disabled())
1658                 return -ENODEV;
1659
1660         /* Make sure that target_freq is within supported range */
1661         if (target_freq > policy->max)
1662                 target_freq = policy->max;
1663         if (target_freq < policy->min)
1664                 target_freq = policy->min;
1665
1666         pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
1667                         policy->cpu, target_freq, relation, old_target_freq);
1668
1669         /*
1670          * This might look like a redundant call as we are checking it again
1671          * after finding index. But it is left intentionally for cases where
1672          * exactly same freq is called again and so we can save on few function
1673          * calls.
1674          */
1675         if (target_freq == policy->cur)
1676                 return 0;
1677
1678         if (cpufreq_driver->target)
1679                 retval = cpufreq_driver->target(policy, target_freq, relation);
1680         else if (cpufreq_driver->target_index) {
1681                 struct cpufreq_frequency_table *freq_table;
1682                 struct cpufreq_freqs freqs;
1683                 bool notify;
1684                 int index;
1685
1686                 freq_table = cpufreq_frequency_get_table(policy->cpu);
1687                 if (unlikely(!freq_table)) {
1688                         pr_err("%s: Unable to find freq_table\n", __func__);
1689                         goto out;
1690                 }
1691
1692                 retval = cpufreq_frequency_table_target(policy, freq_table,
1693                                 target_freq, relation, &index);
1694                 if (unlikely(retval)) {
1695                         pr_err("%s: Unable to find matching freq\n", __func__);
1696                         goto out;
1697                 }
1698
1699                 if (freq_table[index].frequency == policy->cur) {
1700                         retval = 0;
1701                         goto out;
1702                 }
1703
1704                 notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION);
1705
1706                 if (notify) {
1707                         freqs.old = policy->cur;
1708                         freqs.new = freq_table[index].frequency;
1709                         freqs.flags = 0;
1710
1711                         pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n",
1712                                         __func__, policy->cpu, freqs.old,
1713                                         freqs.new);
1714
1715                         cpufreq_notify_transition(policy, &freqs,
1716                                         CPUFREQ_PRECHANGE);
1717                 }
1718
1719                 retval = cpufreq_driver->target_index(policy, index);
1720                 if (retval)
1721                         pr_err("%s: Failed to change cpu frequency: %d\n",
1722                                         __func__, retval);
1723
1724                 if (notify) {
1725                         /*
1726                          * Notify with old freq in case we failed to change
1727                          * frequency
1728                          */
1729                         if (retval)
1730                                 freqs.new = freqs.old;
1731
1732                         cpufreq_notify_transition(policy, &freqs,
1733                                         CPUFREQ_POSTCHANGE);
1734                 }
1735         }
1736
1737 out:
1738         return retval;
1739 }
1740 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1741
1742 int cpufreq_driver_target(struct cpufreq_policy *policy,
1743                           unsigned int target_freq,
1744                           unsigned int relation)
1745 {
1746         int ret = -EINVAL;
1747
1748         down_write(&policy->rwsem);
1749
1750         ret = __cpufreq_driver_target(policy, target_freq, relation);
1751
1752         up_write(&policy->rwsem);
1753
1754         return ret;
1755 }
1756 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
1757
1758 /*
1759  * when "event" is CPUFREQ_GOV_LIMITS
1760  */
1761
1762 static int __cpufreq_governor(struct cpufreq_policy *policy,
1763                                         unsigned int event)
1764 {
1765         int ret;
1766
1767         /* Only must be defined when default governor is known to have latency
1768            restrictions, like e.g. conservative or ondemand.
1769            That this is the case is already ensured in Kconfig
1770         */
1771 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
1772         struct cpufreq_governor *gov = &cpufreq_gov_performance;
1773 #else
1774         struct cpufreq_governor *gov = NULL;
1775 #endif
1776
1777         if (policy->governor->max_transition_latency &&
1778             policy->cpuinfo.transition_latency >
1779             policy->governor->max_transition_latency) {
1780                 if (!gov)
1781                         return -EINVAL;
1782                 else {
1783                         printk(KERN_WARNING "%s governor failed, too long"
1784                                " transition latency of HW, fallback"
1785                                " to %s governor\n",
1786                                policy->governor->name,
1787                                gov->name);
1788                         policy->governor = gov;
1789                 }
1790         }
1791
1792         if (event == CPUFREQ_GOV_POLICY_INIT)
1793                 if (!try_module_get(policy->governor->owner))
1794                         return -EINVAL;
1795
1796         pr_debug("__cpufreq_governor for CPU %u, event %u\n",
1797                                                 policy->cpu, event);
1798
1799         mutex_lock(&cpufreq_governor_lock);
1800         if ((policy->governor_enabled && event == CPUFREQ_GOV_START)
1801             || (!policy->governor_enabled
1802             && (event == CPUFREQ_GOV_LIMITS || event == CPUFREQ_GOV_STOP))) {
1803                 mutex_unlock(&cpufreq_governor_lock);
1804                 return -EBUSY;
1805         }
1806
1807         if (event == CPUFREQ_GOV_STOP)
1808                 policy->governor_enabled = false;
1809         else if (event == CPUFREQ_GOV_START)
1810                 policy->governor_enabled = true;
1811
1812         mutex_unlock(&cpufreq_governor_lock);
1813
1814         ret = policy->governor->governor(policy, event);
1815
1816         if (!ret) {
1817                 if (event == CPUFREQ_GOV_POLICY_INIT)
1818                         policy->governor->initialized++;
1819                 else if (event == CPUFREQ_GOV_POLICY_EXIT)
1820                         policy->governor->initialized--;
1821         } else {
1822                 /* Restore original values */
1823                 mutex_lock(&cpufreq_governor_lock);
1824                 if (event == CPUFREQ_GOV_STOP)
1825                         policy->governor_enabled = true;
1826                 else if (event == CPUFREQ_GOV_START)
1827                         policy->governor_enabled = false;
1828                 mutex_unlock(&cpufreq_governor_lock);
1829         }
1830
1831         if (((event == CPUFREQ_GOV_POLICY_INIT) && ret) ||
1832                         ((event == CPUFREQ_GOV_POLICY_EXIT) && !ret))
1833                 module_put(policy->governor->owner);
1834
1835         return ret;
1836 }
1837
1838 int cpufreq_register_governor(struct cpufreq_governor *governor)
1839 {
1840         int err;
1841
1842         if (!governor)
1843                 return -EINVAL;
1844
1845         if (cpufreq_disabled())
1846                 return -ENODEV;
1847
1848         mutex_lock(&cpufreq_governor_mutex);
1849
1850         governor->initialized = 0;
1851         err = -EBUSY;
1852         if (__find_governor(governor->name) == NULL) {
1853                 err = 0;
1854                 list_add(&governor->governor_list, &cpufreq_governor_list);
1855         }
1856
1857         mutex_unlock(&cpufreq_governor_mutex);
1858         return err;
1859 }
1860 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
1861
1862 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
1863 {
1864 #ifdef CONFIG_HOTPLUG_CPU
1865         int cpu;
1866 #endif
1867
1868         if (!governor)
1869                 return;
1870
1871         if (cpufreq_disabled())
1872                 return;
1873
1874 #ifdef CONFIG_HOTPLUG_CPU
1875         for_each_present_cpu(cpu) {
1876                 if (cpu_online(cpu))
1877                         continue;
1878                 if (!strcmp(per_cpu(cpufreq_cpu_governor, cpu), governor->name))
1879                         strcpy(per_cpu(cpufreq_cpu_governor, cpu), "\0");
1880         }
1881 #endif
1882
1883         mutex_lock(&cpufreq_governor_mutex);
1884         list_del(&governor->governor_list);
1885         mutex_unlock(&cpufreq_governor_mutex);
1886         return;
1887 }
1888 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
1889
1890
1891 /*********************************************************************
1892  *                          POLICY INTERFACE                         *
1893  *********************************************************************/
1894
1895 /**
1896  * cpufreq_get_policy - get the current cpufreq_policy
1897  * @policy: struct cpufreq_policy into which the current cpufreq_policy
1898  *      is written
1899  *
1900  * Reads the current cpufreq policy.
1901  */
1902 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
1903 {
1904         struct cpufreq_policy *cpu_policy;
1905         if (!policy)
1906                 return -EINVAL;
1907
1908         cpu_policy = cpufreq_cpu_get(cpu);
1909         if (!cpu_policy)
1910                 return -EINVAL;
1911
1912         memcpy(policy, cpu_policy, sizeof(*policy));
1913
1914         cpufreq_cpu_put(cpu_policy);
1915         return 0;
1916 }
1917 EXPORT_SYMBOL(cpufreq_get_policy);
1918
1919 /*
1920  * policy : current policy.
1921  * new_policy: policy to be set.
1922  */
1923 static int cpufreq_set_policy(struct cpufreq_policy *policy,
1924                                 struct cpufreq_policy *new_policy)
1925 {
1926         int ret = 0, failed = 1;
1927
1928         pr_debug("setting new policy for CPU %u: %u - %u kHz\n", new_policy->cpu,
1929                 new_policy->min, new_policy->max);
1930
1931         memcpy(&new_policy->cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo));
1932
1933         if (new_policy->min > policy->max || new_policy->max < policy->min) {
1934                 ret = -EINVAL;
1935                 goto error_out;
1936         }
1937
1938         /* verify the cpu speed can be set within this limit */
1939         ret = cpufreq_driver->verify(new_policy);
1940         if (ret)
1941                 goto error_out;
1942
1943         /* adjust if necessary - all reasons */
1944         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1945                         CPUFREQ_ADJUST, new_policy);
1946
1947         /* adjust if necessary - hardware incompatibility*/
1948         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1949                         CPUFREQ_INCOMPATIBLE, new_policy);
1950
1951         /*
1952          * verify the cpu speed can be set within this limit, which might be
1953          * different to the first one
1954          */
1955         ret = cpufreq_driver->verify(new_policy);
1956         if (ret)
1957                 goto error_out;
1958
1959         /* notification of the new policy */
1960         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1961                         CPUFREQ_NOTIFY, new_policy);
1962
1963         policy->min = new_policy->min;
1964         policy->max = new_policy->max;
1965
1966         pr_debug("new min and max freqs are %u - %u kHz\n",
1967                                         policy->min, policy->max);
1968
1969         if (cpufreq_driver->setpolicy) {
1970                 policy->policy = new_policy->policy;
1971                 pr_debug("setting range\n");
1972                 ret = cpufreq_driver->setpolicy(new_policy);
1973         } else {
1974                 if (new_policy->governor != policy->governor) {
1975                         /* save old, working values */
1976                         struct cpufreq_governor *old_gov = policy->governor;
1977
1978                         pr_debug("governor switch\n");
1979
1980                         /* end old governor */
1981                         if (policy->governor) {
1982                                 __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
1983                                 up_write(&policy->rwsem);
1984                                 __cpufreq_governor(policy,
1985                                                 CPUFREQ_GOV_POLICY_EXIT);
1986                                 down_write(&policy->rwsem);
1987                         }
1988
1989                         /* start new governor */
1990                         policy->governor = new_policy->governor;
1991                         if (!__cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT)) {
1992                                 if (!__cpufreq_governor(policy, CPUFREQ_GOV_START)) {
1993                                         failed = 0;
1994                                 } else {
1995                                         up_write(&policy->rwsem);
1996                                         __cpufreq_governor(policy,
1997                                                         CPUFREQ_GOV_POLICY_EXIT);
1998                                         down_write(&policy->rwsem);
1999                                 }
2000                         }
2001
2002                         if (failed) {
2003                                 /* new governor failed, so re-start old one */
2004                                 pr_debug("starting governor %s failed\n",
2005                                                         policy->governor->name);
2006                                 if (old_gov) {
2007                                         policy->governor = old_gov;
2008                                         __cpufreq_governor(policy,
2009                                                         CPUFREQ_GOV_POLICY_INIT);
2010                                         __cpufreq_governor(policy,
2011                                                            CPUFREQ_GOV_START);
2012                                 }
2013                                 ret = -EINVAL;
2014                                 goto error_out;
2015                         }
2016                         /* might be a policy change, too, so fall through */
2017                 }
2018                 pr_debug("governor: change or update limits\n");
2019                 ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
2020         }
2021
2022 error_out:
2023         return ret;
2024 }
2025
2026 /**
2027  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
2028  *      @cpu: CPU which shall be re-evaluated
2029  *
2030  *      Useful for policy notifiers which have different necessities
2031  *      at different times.
2032  */
2033 int cpufreq_update_policy(unsigned int cpu)
2034 {
2035         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
2036         struct cpufreq_policy new_policy;
2037         int ret;
2038
2039         if (!policy) {
2040                 ret = -ENODEV;
2041                 goto no_policy;
2042         }
2043
2044         down_write(&policy->rwsem);
2045
2046         pr_debug("updating policy for CPU %u\n", cpu);
2047         memcpy(&new_policy, policy, sizeof(*policy));
2048         new_policy.min = policy->user_policy.min;
2049         new_policy.max = policy->user_policy.max;
2050         new_policy.policy = policy->user_policy.policy;
2051         new_policy.governor = policy->user_policy.governor;
2052
2053         /*
2054          * BIOS might change freq behind our back
2055          * -> ask driver for current freq and notify governors about a change
2056          */
2057         if (cpufreq_driver->get) {
2058                 new_policy.cur = cpufreq_driver->get(cpu);
2059                 if (!policy->cur) {
2060                         pr_debug("Driver did not initialize current freq");
2061                         policy->cur = new_policy.cur;
2062                 } else {
2063                         if (policy->cur != new_policy.cur && has_target())
2064                                 cpufreq_out_of_sync(cpu, policy->cur,
2065                                                                 new_policy.cur);
2066                 }
2067         }
2068
2069         ret = cpufreq_set_policy(policy, &new_policy);
2070
2071         up_write(&policy->rwsem);
2072
2073         cpufreq_cpu_put(policy);
2074 no_policy:
2075         return ret;
2076 }
2077 EXPORT_SYMBOL(cpufreq_update_policy);
2078
2079 static int cpufreq_cpu_callback(struct notifier_block *nfb,
2080                                         unsigned long action, void *hcpu)
2081 {
2082         unsigned int cpu = (unsigned long)hcpu;
2083         struct device *dev;
2084         bool frozen = false;
2085
2086         dev = get_cpu_device(cpu);
2087         if (dev) {
2088
2089                 if (action & CPU_TASKS_FROZEN)
2090                         frozen = true;
2091
2092                 switch (action & ~CPU_TASKS_FROZEN) {
2093                 case CPU_ONLINE:
2094                         __cpufreq_add_dev(dev, NULL, frozen);
2095                         cpufreq_update_policy(cpu);
2096                         break;
2097
2098                 case CPU_DOWN_PREPARE:
2099                         __cpufreq_remove_dev_prepare(dev, NULL, frozen);
2100                         break;
2101
2102                 case CPU_POST_DEAD:
2103                         __cpufreq_remove_dev_finish(dev, NULL, frozen);
2104                         break;
2105
2106                 case CPU_DOWN_FAILED:
2107                         __cpufreq_add_dev(dev, NULL, frozen);
2108                         break;
2109                 }
2110         }
2111         return NOTIFY_OK;
2112 }
2113
2114 static struct notifier_block __refdata cpufreq_cpu_notifier = {
2115         .notifier_call = cpufreq_cpu_callback,
2116 };
2117
2118 /*********************************************************************
2119  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
2120  *********************************************************************/
2121
2122 /**
2123  * cpufreq_register_driver - register a CPU Frequency driver
2124  * @driver_data: A struct cpufreq_driver containing the values#
2125  * submitted by the CPU Frequency driver.
2126  *
2127  * Registers a CPU Frequency driver to this core code. This code
2128  * returns zero on success, -EBUSY when another driver got here first
2129  * (and isn't unregistered in the meantime).
2130  *
2131  */
2132 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
2133 {
2134         unsigned long flags;
2135         int ret;
2136
2137         if (cpufreq_disabled())
2138                 return -ENODEV;
2139
2140         if (!driver_data || !driver_data->verify || !driver_data->init ||
2141             !(driver_data->setpolicy || driver_data->target_index ||
2142                     driver_data->target))
2143                 return -EINVAL;
2144
2145         pr_debug("trying to register driver %s\n", driver_data->name);
2146
2147         if (driver_data->setpolicy)
2148                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
2149
2150         write_lock_irqsave(&cpufreq_driver_lock, flags);
2151         if (cpufreq_driver) {
2152                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2153                 return -EEXIST;
2154         }
2155         cpufreq_driver = driver_data;
2156         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2157
2158         ret = subsys_interface_register(&cpufreq_interface);
2159         if (ret)
2160                 goto err_null_driver;
2161
2162         if (!(cpufreq_driver->flags & CPUFREQ_STICKY)) {
2163                 int i;
2164                 ret = -ENODEV;
2165
2166                 /* check for at least one working CPU */
2167                 for (i = 0; i < nr_cpu_ids; i++)
2168                         if (cpu_possible(i) && per_cpu(cpufreq_cpu_data, i)) {
2169                                 ret = 0;
2170                                 break;
2171                         }
2172
2173                 /* if all ->init() calls failed, unregister */
2174                 if (ret) {
2175                         pr_debug("no CPU initialized for driver %s\n",
2176                                                         driver_data->name);
2177                         goto err_if_unreg;
2178                 }
2179         }
2180
2181         register_hotcpu_notifier(&cpufreq_cpu_notifier);
2182         pr_debug("driver %s up and running\n", driver_data->name);
2183
2184         return 0;
2185 err_if_unreg:
2186         subsys_interface_unregister(&cpufreq_interface);
2187 err_null_driver:
2188         write_lock_irqsave(&cpufreq_driver_lock, flags);
2189         cpufreq_driver = NULL;
2190         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2191         return ret;
2192 }
2193 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
2194
2195 /**
2196  * cpufreq_unregister_driver - unregister the current CPUFreq driver
2197  *
2198  * Unregister the current CPUFreq driver. Only call this if you have
2199  * the right to do so, i.e. if you have succeeded in initialising before!
2200  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
2201  * currently not initialised.
2202  */
2203 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
2204 {
2205         unsigned long flags;
2206
2207         if (!cpufreq_driver || (driver != cpufreq_driver))
2208                 return -EINVAL;
2209
2210         pr_debug("unregistering driver %s\n", driver->name);
2211
2212         subsys_interface_unregister(&cpufreq_interface);
2213         unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
2214
2215         down_write(&cpufreq_rwsem);
2216         write_lock_irqsave(&cpufreq_driver_lock, flags);
2217
2218         cpufreq_driver = NULL;
2219
2220         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2221         up_write(&cpufreq_rwsem);
2222
2223         return 0;
2224 }
2225 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
2226
2227 static int __init cpufreq_core_init(void)
2228 {
2229         if (cpufreq_disabled())
2230                 return -ENODEV;
2231
2232         cpufreq_global_kobject = kobject_create();
2233         BUG_ON(!cpufreq_global_kobject);
2234         register_syscore_ops(&cpufreq_syscore_ops);
2235
2236         return 0;
2237 }
2238 core_initcall(cpufreq_core_init);