drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6  *            (C) 2013 Viresh Kumar <viresh.kumar@linaro.org>
   7  *
   8  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   9  *      Added handling for CPU hotplug
  10  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  11  *      Fix handling for CPU hotplug -- affected CPUs
  12  *
  13  * This program is free software; you can redistribute it and/or modify
  14  * it under the terms of the GNU General Public License version 2 as
  15  * published by the Free Software Foundation.
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <asm/cputime.h>
  21 #include <linux/kernel.h>
  22 #include <linux/kernel_stat.h>
  23 #include <linux/module.h>
  24 #include <linux/init.h>
  25 #include <linux/notifier.h>
  26 #include <linux/cpufreq.h>
  27 #include <linux/delay.h>
  28 #include <linux/interrupt.h>
  29 #include <linux/spinlock.h>
  30 #include <linux/tick.h>
  31 #include <linux/device.h>
  32 #include <linux/slab.h>
  33 #include <linux/cpu.h>
  34 #include <linux/completion.h>
  35 #include <linux/mutex.h>
  36 #include <linux/syscore_ops.h>
  37
  38 #include <trace/events/power.h>
  39
  40 /**
  41  * The "cpufreq driver" - the arch- or hardware-dependent low
  42  * level driver of CPUFreq support, and its spinlock. This lock
  43  * also protects the cpufreq_cpu_data array.
  44  */
  45 static struct cpufreq_driver *cpufreq_driver;
  46 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
  47 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data_fallback);
  48 static DEFINE_RWLOCK(cpufreq_driver_lock);
  49 static DEFINE_MUTEX(cpufreq_governor_lock);
  50
  51 #ifdef CONFIG_HOTPLUG_CPU
  52 /* This one keeps track of the previously set governor of a removed CPU */
  53 static DEFINE_PER_CPU(char[CPUFREQ_NAME_LEN], cpufreq_cpu_governor);
  54 #endif
  55
  56 /*
  57  * cpu_policy_rwsem is a per CPU reader-writer semaphore designed to cure
  58  * all cpufreq/hotplug/workqueue/etc related lock issues.
  59  *
  60  * The rules for this semaphore:
  61  * - Any routine that wants to read from the policy structure will
  62  *   do a down_read on this semaphore.
  63  * - Any routine that will write to the policy structure and/or may take away
  64  *   the policy altogether (eg. CPU hotplug), will hold this lock in write
  65  *   mode before doing so.
  66  *
  67  * Additional rules:
  68  * - Governor routines that can be called in cpufreq hotplug path should not
  69  *   take this sem as top level hotplug notifier handler takes this.
  70  * - Lock should not be held across
  71  *     __cpufreq_governor(data, CPUFREQ_GOV_STOP);
  72  */
  73 static DEFINE_PER_CPU(int, cpufreq_policy_cpu);
  74 static DEFINE_PER_CPU(struct rw_semaphore, cpu_policy_rwsem);
  75
  76 #define lock_policy_rwsem(mode, cpu)                                    \
  77 static int lock_policy_rwsem_##mode(int cpu)                            \
  78 {                                                                       \
  79         int policy_cpu = per_cpu(cpufreq_policy_cpu, cpu);              \
  80         BUG_ON(policy_cpu == -1);                                       \
  81         down_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));            \
  82                                                                         \
  83         return 0;                                                       \
  84 }
  85
  86 lock_policy_rwsem(read, cpu);
  87 lock_policy_rwsem(write, cpu);
  88
  89 #define unlock_policy_rwsem(mode, cpu)                                  \
  90 static void unlock_policy_rwsem_##mode(int cpu)                         \
  91 {                                                                       \
  92         int policy_cpu = per_cpu(cpufreq_policy_cpu, cpu);              \
  93         BUG_ON(policy_cpu == -1);                                       \
  94         up_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));              \
  95 }
  96
  97 unlock_policy_rwsem(read, cpu);
  98 unlock_policy_rwsem(write, cpu);
  99
 100 /* internal prototypes */
 101 static int __cpufreq_governor(struct cpufreq_policy *policy,
 102                 unsigned int event);
 103 static unsigned int __cpufreq_get(unsigned int cpu);
 104 static void handle_update(struct work_struct *work);
 105
 106 /**
 107  * Two notifier lists: the "policy" list is involved in the
 108  * validation process for a new CPU frequency policy; the
 109  * "transition" list for kernel code that needs to handle
 110  * changes to devices when the CPU clock speed changes.
 111  * The mutex locks both lists.
 112  */
 113 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
 114 static struct srcu_notifier_head cpufreq_transition_notifier_list;
 115
 116 static bool init_cpufreq_transition_notifier_list_called;
 117 static int __init init_cpufreq_transition_notifier_list(void)
 118 {
 119         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
 120         init_cpufreq_transition_notifier_list_called = true;
 121         return 0;
 122 }
 123 pure_initcall(init_cpufreq_transition_notifier_list);
 124
 125 static int off __read_mostly;
 126 static int cpufreq_disabled(void)
 127 {
 128         return off;
 129 }
 130 void disable_cpufreq(void)
 131 {
 132         off = 1;
 133 }
 134 static LIST_HEAD(cpufreq_governor_list);
 135 static DEFINE_MUTEX(cpufreq_governor_mutex);
 136
 137 bool have_governor_per_policy(void)
 138 {
 139         return cpufreq_driver->have_governor_per_policy;
 140 }
 141 EXPORT_SYMBOL_GPL(have_governor_per_policy);
 142
 143 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
 144 {
 145         if (have_governor_per_policy())
 146                 return &policy->kobj;
 147         else
 148                 return cpufreq_global_kobject;
 149 }
 150 EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
 151
 152 static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
 153 {
 154         u64 idle_time;
 155         u64 cur_wall_time;
 156         u64 busy_time;
 157
 158         cur_wall_time = jiffies64_to_cputime64(get_jiffies_64());
 159
 160         busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER];
 161         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM];
 162         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ];
 163         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ];
 164         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL];
 165         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE];
 166
 167         idle_time = cur_wall_time - busy_time;
 168         if (wall)
 169                 *wall = cputime_to_usecs(cur_wall_time);
 170
 171         return cputime_to_usecs(idle_time);
 172 }
 173
 174 u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
 175 {
 176         u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);
 177
 178         if (idle_time == -1ULL)
 179                 return get_cpu_idle_time_jiffy(cpu, wall);
 180         else if (!io_busy)
 181                 idle_time += get_cpu_iowait_time_us(cpu, wall);
 182
 183         return idle_time;
 184 }
 185 EXPORT_SYMBOL_GPL(get_cpu_idle_time);
 186
 187 static struct cpufreq_policy *__cpufreq_cpu_get(unsigned int cpu, bool sysfs)
 188 {
 189         struct cpufreq_policy *data;
 190         unsigned long flags;
 191
 192         if (cpu >= nr_cpu_ids)
 193                 goto err_out;
 194
 195         /* get the cpufreq driver */
 196         read_lock_irqsave(&cpufreq_driver_lock, flags);
 197
 198         if (!cpufreq_driver)
 199                 goto err_out_unlock;
 200
 201         if (!try_module_get(cpufreq_driver->owner))
 202                 goto err_out_unlock;
 203
 204         /* get the CPU */
 205         data = per_cpu(cpufreq_cpu_data, cpu);
 206
 207         if (!data)
 208                 goto err_out_put_module;
 209
 210         if (!sysfs && !kobject_get(&data->kobj))
 211                 goto err_out_put_module;
 212
 213         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 214         return data;
 215
 216 err_out_put_module:
 217         module_put(cpufreq_driver->owner);
 218 err_out_unlock:
 219         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 220 err_out:
 221         return NULL;
 222 }
 223
 224 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 225 {
 226         if (cpufreq_disabled())
 227                 return NULL;
 228
 229         return __cpufreq_cpu_get(cpu, false);
 230 }
 231 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 232
 233 static struct cpufreq_policy *cpufreq_cpu_get_sysfs(unsigned int cpu)
 234 {
 235         return __cpufreq_cpu_get(cpu, true);
 236 }
 237
 238 static void __cpufreq_cpu_put(struct cpufreq_policy *data, bool sysfs)
 239 {
 240         if (!sysfs)
 241                 kobject_put(&data->kobj);
 242         module_put(cpufreq_driver->owner);
 243 }
 244
 245 void cpufreq_cpu_put(struct cpufreq_policy *data)
 246 {
 247         if (cpufreq_disabled())
 248                 return;
 249
 250         __cpufreq_cpu_put(data, false);
 251 }
 252 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 253
 254 static void cpufreq_cpu_put_sysfs(struct cpufreq_policy *data)
 255 {
 256         __cpufreq_cpu_put(data, true);
 257 }
 258
 259 /*********************************************************************
 260  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 261  *********************************************************************/
 262
 263 /**
 264  * adjust_jiffies - adjust the system "loops_per_jiffy"
 265  *
 266  * This function alters the system "loops_per_jiffy" for the clock
 267  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 268  * systems as each CPU might be scaled differently. So, use the arch
 269  * per-CPU loops_per_jiffy value wherever possible.
 270  */
 271 #ifndef CONFIG_SMP
 272 static unsigned long l_p_j_ref;
 273 static unsigned int l_p_j_ref_freq;
 274
 275 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 276 {
 277         if (ci->flags & CPUFREQ_CONST_LOOPS)
 278                 return;
 279
 280         if (!l_p_j_ref_freq) {
 281                 l_p_j_ref = loops_per_jiffy;
 282                 l_p_j_ref_freq = ci->old;
 283                 pr_debug("saving %lu as reference value for loops_per_jiffy; "
 284                         "freq is %u kHz\n", l_p_j_ref, l_p_j_ref_freq);
 285         }
 286         if ((val == CPUFREQ_POSTCHANGE && ci->old != ci->new) ||
 287             (val == CPUFREQ_RESUMECHANGE || val == CPUFREQ_SUSPENDCHANGE)) {
 288                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 289                                                                 ci->new);
 290                 pr_debug("scaling loops_per_jiffy to %lu "
 291                         "for frequency %u kHz\n", loops_per_jiffy, ci->new);
 292         }
 293 }
 294 #else
 295 static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 296 {
 297         return;
 298 }
 299 #endif
 300
 301 static void __cpufreq_notify_transition(struct cpufreq_policy *policy,
 302                 struct cpufreq_freqs *freqs, unsigned int state)
 303 {
 304         BUG_ON(irqs_disabled());
 305
 306         if (cpufreq_disabled())
 307                 return;
 308
 309         freqs->flags = cpufreq_driver->flags;
 310         pr_debug("notification %u of frequency transition to %u kHz\n",
 311                 state, freqs->new);
 312
 313         switch (state) {
 314
 315         case CPUFREQ_PRECHANGE:
 316                 if (WARN(policy->transition_ongoing ==
 317                                         cpumask_weight(policy->cpus),
 318                                 "In middle of another frequency transition\n"))
 319                         return;
 320
 321                 policy->transition_ongoing++;
 322
 323                 /* detect if the driver reported a value as "old frequency"
 324                  * which is not equal to what the cpufreq core thinks is
 325                  * "old frequency".
 326                  */
 327                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 328                         if ((policy) && (policy->cpu == freqs->cpu) &&
 329                             (policy->cur) && (policy->cur != freqs->old)) {
 330                                 pr_debug("Warning: CPU frequency is"
 331                                         " %u, cpufreq assumed %u kHz.\n",
 332                                         freqs->old, policy->cur);
 333                                 freqs->old = policy->cur;
 334                         }
 335                 }
 336                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 337                                 CPUFREQ_PRECHANGE, freqs);
 338                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 339                 break;
 340
 341         case CPUFREQ_POSTCHANGE:
 342                 if (WARN(!policy->transition_ongoing,
 343                                 "No frequency transition in progress\n"))
 344                         return;
 345
 346                 policy->transition_ongoing--;
 347
 348                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 349                 pr_debug("FREQ: %lu - CPU: %lu", (unsigned long)freqs->new,
 350                         (unsigned long)freqs->cpu);
 351                 trace_cpu_frequency(freqs->new, freqs->cpu);
 352                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 353                                 CPUFREQ_POSTCHANGE, freqs);
 354                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 355                         policy->cur = freqs->new;
 356                 break;
 357         }
 358 }
 359
 360 /**
 361  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 362  * on frequency transition.
 363  *
 364  * This function calls the transition notifiers and the "adjust_jiffies"
 365  * function. It is called twice on all CPU frequency changes that have
 366  * external effects.
 367  */
 368 void cpufreq_notify_transition(struct cpufreq_policy *policy,
 369                 struct cpufreq_freqs *freqs, unsigned int state)
 370 {
 371         for_each_cpu(freqs->cpu, policy->cpus)
 372                 __cpufreq_notify_transition(policy, freqs, state);
 373 }
 374 EXPORT_SYMBOL_GPL(cpufreq_notify_transition);
 375
 376
 377 /*********************************************************************
 378  *                          SYSFS INTERFACE                          *
 379  *********************************************************************/
 380
 381 static struct cpufreq_governor *__find_governor(const char *str_governor)
 382 {
 383         struct cpufreq_governor *t;
 384
 385         list_for_each_entry(t, &cpufreq_governor_list, governor_list)
 386                 if (!strnicmp(str_governor, t->name, CPUFREQ_NAME_LEN))
 387                         return t;
 388
 389         return NULL;
 390 }
 391
 392 /**
 393  * cpufreq_parse_governor - parse a governor string
 394  */
 395 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
 396                                 struct cpufreq_governor **governor)
 397 {
 398         int err = -EINVAL;
 399
 400         if (!cpufreq_driver)
 401                 goto out;
 402
 403         if (cpufreq_driver->setpolicy) {
 404                 if (!strnicmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 405                         *policy = CPUFREQ_POLICY_PERFORMANCE;
 406                         err = 0;
 407                 } else if (!strnicmp(str_governor, "powersave",
 408                                                 CPUFREQ_NAME_LEN)) {
 409                         *policy = CPUFREQ_POLICY_POWERSAVE;
 410                         err = 0;
 411                 }
 412         } else if (cpufreq_driver->target) {
 413                 struct cpufreq_governor *t;
 414
 415                 mutex_lock(&cpufreq_governor_mutex);
 416
 417                 t = __find_governor(str_governor);
 418
 419                 if (t == NULL) {
 420                         int ret;
 421
 422                         mutex_unlock(&cpufreq_governor_mutex);
 423                         ret = request_module("cpufreq_%s", str_governor);
 424                         mutex_lock(&cpufreq_governor_mutex);
 425
 426                         if (ret == 0)
 427                                 t = __find_governor(str_governor);
 428                 }
 429
 430                 if (t != NULL) {
 431                         *governor = t;
 432                         err = 0;
 433                 }
 434
 435                 mutex_unlock(&cpufreq_governor_mutex);
 436         }
 437 out:
 438         return err;
 439 }
 440
 441 /**
 442  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 443  * print out cpufreq information
 444  *
 445  * Write out information from cpufreq_driver->policy[cpu]; object must be
 446  * "unsigned int".
 447  */
 448
 449 #define show_one(file_name, object)                     \
 450 static ssize_t show_##file_name                         \
 451 (struct cpufreq_policy *policy, char *buf)              \
 452 {                                                       \
 453         return sprintf(buf, "%u\n", policy->object);    \
 454 }
 455
 456 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 457 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 458 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
 459 show_one(scaling_min_freq, min);
 460 show_one(scaling_max_freq, max);
 461 show_one(scaling_cur_freq, cur);
 462
 463 static int __cpufreq_set_policy(struct cpufreq_policy *data,
 464                                 struct cpufreq_policy *policy);
 465
 466 /**
 467  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 468  */
 469 #define store_one(file_name, object)                    \
 470 static ssize_t store_##file_name                                        \
 471 (struct cpufreq_policy *policy, const char *buf, size_t count)          \
 472 {                                                                       \
 473         unsigned int ret;                                               \
 474         struct cpufreq_policy new_policy;                               \
 475                                                                         \
 476         ret = cpufreq_get_policy(&new_policy, policy->cpu);             \
 477         if (ret)                                                        \
 478                 return -EINVAL;                                         \
 479                                                                         \
 480         ret = sscanf(buf, "%u", &new_policy.object);                    \
 481         if (ret != 1)                                                   \
 482                 return -EINVAL;                                         \
 483                                                                         \
 484         ret = __cpufreq_set_policy(policy, &new_policy);                \
 485         policy->user_policy.object = policy->object;                    \
 486                                                                         \
 487         return ret ? ret : count;                                       \
 488 }
 489
 490 store_one(scaling_min_freq, min);
 491 store_one(scaling_max_freq, max);
 492
 493 /**
 494  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 495  */
 496 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
 497                                         char *buf)
 498 {
 499         unsigned int cur_freq = __cpufreq_get(policy->cpu);
 500         if (!cur_freq)
 501                 return sprintf(buf, "<unknown>");
 502         return sprintf(buf, "%u\n", cur_freq);
 503 }
 504
 505 /**
 506  * show_scaling_governor - show the current policy for the specified CPU
 507  */
 508 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
 509 {
 510         if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
 511                 return sprintf(buf, "powersave\n");
 512         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 513                 return sprintf(buf, "performance\n");
 514         else if (policy->governor)
 515                 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
 516                                 policy->governor->name);
 517         return -EINVAL;
 518 }
 519
 520 /**
 521  * store_scaling_governor - store policy for the specified CPU
 522  */
 523 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
 524                                         const char *buf, size_t count)
 525 {
 526         unsigned int ret;
 527         char    str_governor[16];
 528         struct cpufreq_policy new_policy;
 529
 530         ret = cpufreq_get_policy(&new_policy, policy->cpu);
 531         if (ret)
 532                 return ret;
 533
 534         ret = sscanf(buf, "%15s", str_governor);
 535         if (ret != 1)
 536                 return -EINVAL;
 537
 538         if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 539                                                 &new_policy.governor))
 540                 return -EINVAL;
 541
 542         /*
 543          * Do not use cpufreq_set_policy here or the user_policy.max
 544          * will be wrongly overridden
 545          */
 546         ret = __cpufreq_set_policy(policy, &new_policy);
 547
 548         policy->user_policy.policy = policy->policy;
 549         policy->user_policy.governor = policy->governor;
 550
 551         if (ret)
 552                 return ret;
 553         else
 554                 return count;
 555 }
 556
 557 /**
 558  * show_scaling_driver - show the cpufreq driver currently loaded
 559  */
 560 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
 561 {
 562         return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
 563 }
 564
 565 /**
 566  * show_scaling_available_governors - show the available CPUfreq governors
 567  */
 568 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
 569                                                 char *buf)
 570 {
 571         ssize_t i = 0;
 572         struct cpufreq_governor *t;
 573
 574         if (!cpufreq_driver->target) {
 575                 i += sprintf(buf, "performance powersave");
 576                 goto out;
 577         }
 578
 579         list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
 580                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
 581                     - (CPUFREQ_NAME_LEN + 2)))
 582                         goto out;
 583                 i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
 584         }
 585 out:
 586         i += sprintf(&buf[i], "\n");
 587         return i;
 588 }
 589
 590 ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
 591 {
 592         ssize_t i = 0;
 593         unsigned int cpu;
 594
 595         for_each_cpu(cpu, mask) {
 596                 if (i)
 597                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 598                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 599                 if (i >= (PAGE_SIZE - 5))
 600                         break;
 601         }
 602         i += sprintf(&buf[i], "\n");
 603         return i;
 604 }
 605 EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
 606
 607 /**
 608  * show_related_cpus - show the CPUs affected by each transition even if
 609  * hw coordination is in use
 610  */
 611 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
 612 {
 613         return cpufreq_show_cpus(policy->related_cpus, buf);
 614 }
 615
 616 /**
 617  * show_affected_cpus - show the CPUs affected by each transition
 618  */
 619 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
 620 {
 621         return cpufreq_show_cpus(policy->cpus, buf);
 622 }
 623
 624 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 625                                         const char *buf, size_t count)
 626 {
 627         unsigned int freq = 0;
 628         unsigned int ret;
 629
 630         if (!policy->governor || !policy->governor->store_setspeed)
 631                 return -EINVAL;
 632
 633         ret = sscanf(buf, "%u", &freq);
 634         if (ret != 1)
 635                 return -EINVAL;
 636
 637         policy->governor->store_setspeed(policy, freq);
 638
 639         return count;
 640 }
 641
 642 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 643 {
 644         if (!policy->governor || !policy->governor->show_setspeed)
 645                 return sprintf(buf, "<unsupported>\n");
 646
 647         return policy->governor->show_setspeed(policy, buf);
 648 }
 649
 650 /**
 651  * show_bios_limit - show the current cpufreq HW/BIOS limitation
 652  */
 653 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
 654 {
 655         unsigned int limit;
 656         int ret;
 657         if (cpufreq_driver->bios_limit) {
 658                 ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
 659                 if (!ret)
 660                         return sprintf(buf, "%u\n", limit);
 661         }
 662         return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
 663 }
 664
 665 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
 666 cpufreq_freq_attr_ro(cpuinfo_min_freq);
 667 cpufreq_freq_attr_ro(cpuinfo_max_freq);
 668 cpufreq_freq_attr_ro(cpuinfo_transition_latency);
 669 cpufreq_freq_attr_ro(scaling_available_governors);
 670 cpufreq_freq_attr_ro(scaling_driver);
 671 cpufreq_freq_attr_ro(scaling_cur_freq);
 672 cpufreq_freq_attr_ro(bios_limit);
 673 cpufreq_freq_attr_ro(related_cpus);
 674 cpufreq_freq_attr_ro(affected_cpus);
 675 cpufreq_freq_attr_rw(scaling_min_freq);
 676 cpufreq_freq_attr_rw(scaling_max_freq);
 677 cpufreq_freq_attr_rw(scaling_governor);
 678 cpufreq_freq_attr_rw(scaling_setspeed);
 679
 680 static struct attribute *default_attrs[] = {
 681         &cpuinfo_min_freq.attr,
 682         &cpuinfo_max_freq.attr,
 683         &cpuinfo_transition_latency.attr,
 684         &scaling_min_freq.attr,
 685         &scaling_max_freq.attr,
 686         &affected_cpus.attr,
 687         &related_cpus.attr,
 688         &scaling_governor.attr,
 689         &scaling_driver.attr,
 690         &scaling_available_governors.attr,
 691         &scaling_setspeed.attr,
 692         NULL
 693 };
 694
 695 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
 696 #define to_attr(a) container_of(a, struct freq_attr, attr)
 697
 698 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
 699 {
 700         struct cpufreq_policy *policy = to_policy(kobj);
 701         struct freq_attr *fattr = to_attr(attr);
 702         ssize_t ret = -EINVAL;
 703         policy = cpufreq_cpu_get_sysfs(policy->cpu);
 704         if (!policy)
 705                 goto no_policy;
 706
 707         if (lock_policy_rwsem_read(policy->cpu) < 0)
 708                 goto fail;
 709
 710         if (fattr->show)
 711                 ret = fattr->show(policy, buf);
 712         else
 713                 ret = -EIO;
 714
 715         unlock_policy_rwsem_read(policy->cpu);
 716 fail:
 717         cpufreq_cpu_put_sysfs(policy);
 718 no_policy:
 719         return ret;
 720 }
 721
 722 static ssize_t store(struct kobject *kobj, struct attribute *attr,
 723                      const char *buf, size_t count)
 724 {
 725         struct cpufreq_policy *policy = to_policy(kobj);
 726         struct freq_attr *fattr = to_attr(attr);
 727         ssize_t ret = -EINVAL;
 728         policy = cpufreq_cpu_get_sysfs(policy->cpu);
 729         if (!policy)
 730                 goto no_policy;
 731
 732         if (lock_policy_rwsem_write(policy->cpu) < 0)
 733                 goto fail;
 734
 735         if (fattr->store)
 736                 ret = fattr->store(policy, buf, count);
 737         else
 738                 ret = -EIO;
 739
 740         unlock_policy_rwsem_write(policy->cpu);
 741 fail:
 742         cpufreq_cpu_put_sysfs(policy);
 743 no_policy:
 744         return ret;
 745 }
 746
 747 static void cpufreq_sysfs_release(struct kobject *kobj)
 748 {
 749         struct cpufreq_policy *policy = to_policy(kobj);
 750         pr_debug("last reference is dropped\n");
 751         complete(&policy->kobj_unregister);
 752 }
 753
 754 static const struct sysfs_ops sysfs_ops = {
 755         .show   = show,
 756         .store  = store,
 757 };
 758
 759 static struct kobj_type ktype_cpufreq = {
 760         .sysfs_ops      = &sysfs_ops,
 761         .default_attrs  = default_attrs,
 762         .release        = cpufreq_sysfs_release,
 763 };
 764
 765 struct kobject *cpufreq_global_kobject;
 766 EXPORT_SYMBOL(cpufreq_global_kobject);
 767
 768 static int cpufreq_global_kobject_usage;
 769
 770 int cpufreq_get_global_kobject(void)
 771 {
 772         if (!cpufreq_global_kobject_usage++)
 773                 return kobject_add(cpufreq_global_kobject,
 774                                 &cpu_subsys.dev_root->kobj, "%s", "cpufreq");
 775
 776         return 0;
 777 }
 778 EXPORT_SYMBOL(cpufreq_get_global_kobject);
 779
 780 void cpufreq_put_global_kobject(void)
 781 {
 782         if (!--cpufreq_global_kobject_usage)
 783                 kobject_del(cpufreq_global_kobject);
 784 }
 785 EXPORT_SYMBOL(cpufreq_put_global_kobject);
 786
 787 int cpufreq_sysfs_create_file(const struct attribute *attr)
 788 {
 789         int ret = cpufreq_get_global_kobject();
 790
 791         if (!ret) {
 792                 ret = sysfs_create_file(cpufreq_global_kobject, attr);
 793                 if (ret)
 794                         cpufreq_put_global_kobject();
 795         }
 796
 797         return ret;
 798 }
 799 EXPORT_SYMBOL(cpufreq_sysfs_create_file);
 800
 801 void cpufreq_sysfs_remove_file(const struct attribute *attr)
 802 {
 803         sysfs_remove_file(cpufreq_global_kobject, attr);
 804         cpufreq_put_global_kobject();
 805 }
 806 EXPORT_SYMBOL(cpufreq_sysfs_remove_file);
 807
 808 /* symlink affected CPUs */
 809 static int cpufreq_add_dev_symlink(struct cpufreq_policy *policy)
 810 {
 811         unsigned int j;
 812         int ret = 0;
 813
 814         for_each_cpu(j, policy->cpus) {
 815                 struct device *cpu_dev;
 816
 817                 if (j == policy->cpu)
 818                         continue;
 819
 820                 pr_debug("Adding link for CPU: %u\n", j);
 821                 cpu_dev = get_cpu_device(j);
 822                 ret = sysfs_create_link(&cpu_dev->kobj, &policy->kobj,
 823                                         "cpufreq");
 824                 if (ret)
 825                         break;
 826         }
 827         return ret;
 828 }
 829
 830 static int cpufreq_add_dev_interface(struct cpufreq_policy *policy,
 831                                      struct device *dev)
 832 {
 833         struct freq_attr **drv_attr;
 834         int ret = 0;
 835
 836         /* prepare interface data */
 837         ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
 838                                    &dev->kobj, "cpufreq");
 839         if (ret)
 840                 return ret;
 841
 842         /* set up files for this cpu device */
 843         drv_attr = cpufreq_driver->attr;
 844         while ((drv_attr) && (*drv_attr)) {
 845                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
 846                 if (ret)
 847                         goto err_out_kobj_put;
 848                 drv_attr++;
 849         }
 850         if (cpufreq_driver->get) {
 851                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
 852                 if (ret)
 853                         goto err_out_kobj_put;
 854         }
 855         if (cpufreq_driver->target) {
 856                 ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
 857                 if (ret)
 858                         goto err_out_kobj_put;
 859         }
 860         if (cpufreq_driver->bios_limit) {
 861                 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
 862                 if (ret)
 863                         goto err_out_kobj_put;
 864         }
 865
 866         ret = cpufreq_add_dev_symlink(policy);
 867         if (ret)
 868                 goto err_out_kobj_put;
 869
 870         return ret;
 871
 872 err_out_kobj_put:
 873         kobject_put(&policy->kobj);
 874         wait_for_completion(&policy->kobj_unregister);
 875         return ret;
 876 }
 877
 878 static void cpufreq_init_policy(struct cpufreq_policy *policy)
 879 {
 880         struct cpufreq_policy new_policy;
 881         int ret = 0;
 882
 883         memcpy(&new_policy, policy, sizeof(struct cpufreq_policy));
 884         /* assure that the starting sequence is run in __cpufreq_set_policy */
 885         policy->governor = NULL;
 886
 887         /* set default policy */
 888         ret = __cpufreq_set_policy(policy, &new_policy);
 889         policy->user_policy.policy = policy->policy;
 890         policy->user_policy.governor = policy->governor;
 891
 892         if (ret) {
 893                 pr_debug("setting policy failed\n");
 894                 if (cpufreq_driver->exit)
 895                         cpufreq_driver->exit(policy);
 896         }
 897 }
 898
 899 #ifdef CONFIG_HOTPLUG_CPU
 900 static int cpufreq_add_policy_cpu(unsigned int cpu, unsigned int sibling,
 901                                   struct device *dev, bool frozen)
 902 {
 903         struct cpufreq_policy *policy;
 904         int ret = 0, has_target = !!cpufreq_driver->target;
 905         unsigned long flags;
 906
 907         policy = cpufreq_cpu_get(sibling);
 908         if (WARN_ON_ONCE(!policy))
 909                 return -ENODATA;
 910
 911         if (has_target)
 912                 __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
 913
 914         lock_policy_rwsem_write(sibling);
 915
 916         write_lock_irqsave(&cpufreq_driver_lock, flags);
 917
 918         cpumask_set_cpu(cpu, policy->cpus);
 919         per_cpu(cpufreq_policy_cpu, cpu) = policy->cpu;
 920         per_cpu(cpufreq_cpu_data, cpu) = policy;
 921         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
 922
 923         unlock_policy_rwsem_write(sibling);
 924
 925         if (has_target) {
 926                 __cpufreq_governor(policy, CPUFREQ_GOV_START);
 927                 __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
 928         }
 929
 930         /* Don't touch sysfs links during light-weight init */
 931         if (!frozen)
 932                 ret = sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq");
 933
 934         cpufreq_cpu_put(policy);
 935         return ret;
 936 }
 937 #endif
 938
 939 static struct cpufreq_policy *cpufreq_policy_restore(unsigned int cpu)
 940 {
 941         struct cpufreq_policy *policy;
 942         unsigned long flags;
 943
 944         write_lock_irqsave(&cpufreq_driver_lock, flags);
 945
 946         policy = per_cpu(cpufreq_cpu_data_fallback, cpu);
 947
 948         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
 949
 950         return policy;
 951 }
 952
 953 static struct cpufreq_policy *cpufreq_policy_alloc(void)
 954 {
 955         struct cpufreq_policy *policy;
 956
 957         policy = kzalloc(sizeof(*policy), GFP_KERNEL);
 958         if (!policy)
 959                 return NULL;
 960
 961         if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
 962                 goto err_free_policy;
 963
 964         if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
 965                 goto err_free_cpumask;
 966
 967         return policy;
 968
 969 err_free_cpumask:
 970         free_cpumask_var(policy->cpus);
 971 err_free_policy:
 972         kfree(policy);
 973
 974         return NULL;
 975 }
 976
 977 static void cpufreq_policy_free(struct cpufreq_policy *policy)
 978 {
 979         free_cpumask_var(policy->related_cpus);
 980         free_cpumask_var(policy->cpus);
 981         kfree(policy);
 982 }
 983
 984 static int __cpufreq_add_dev(struct device *dev, struct subsys_interface *sif,
 985                              bool frozen)
 986 {
 987         unsigned int j, cpu = dev->id;
 988         int ret = -ENOMEM;
 989         struct cpufreq_policy *policy;
 990         unsigned long flags;
 991 #ifdef CONFIG_HOTPLUG_CPU
 992         struct cpufreq_governor *gov;
 993         int sibling;
 994 #endif
 995
 996         if (cpu_is_offline(cpu))
 997                 return 0;
 998
 999         pr_debug("adding CPU %u\n", cpu);
1000
1001 #ifdef CONFIG_SMP
1002         /* check whether a different CPU already registered this
1003          * CPU because it is in the same boat. */
1004         policy = cpufreq_cpu_get(cpu);
1005         if (unlikely(policy)) {
1006                 cpufreq_cpu_put(policy);
1007                 return 0;
1008         }
1009
1010 #ifdef CONFIG_HOTPLUG_CPU
1011         /* Check if this cpu was hot-unplugged earlier and has siblings */
1012         read_lock_irqsave(&cpufreq_driver_lock, flags);
1013         for_each_online_cpu(sibling) {
1014                 struct cpufreq_policy *cp = per_cpu(cpufreq_cpu_data, sibling);
1015                 if (cp && cpumask_test_cpu(cpu, cp->related_cpus)) {
1016                         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1017                         return cpufreq_add_policy_cpu(cpu, sibling, dev,
1018                                                       frozen);
1019                 }
1020         }
1021         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1022 #endif
1023 #endif
1024
1025         if (!try_module_get(cpufreq_driver->owner)) {
1026                 ret = -EINVAL;
1027                 goto module_out;
1028         }
1029
1030         if (frozen)
1031                 /* Restore the saved policy when doing light-weight init */
1032                 policy = cpufreq_policy_restore(cpu);
1033         else
1034                 policy = cpufreq_policy_alloc();
1035
1036         if (!policy)
1037                 goto nomem_out;
1038
1039         policy->cpu = cpu;
1040         policy->governor = CPUFREQ_DEFAULT_GOVERNOR;
1041         cpumask_copy(policy->cpus, cpumask_of(cpu));
1042
1043         /* Initially set CPU itself as the policy_cpu */
1044         per_cpu(cpufreq_policy_cpu, cpu) = cpu;
1045
1046         init_completion(&policy->kobj_unregister);
1047         INIT_WORK(&policy->update, handle_update);
1048
1049         /* call driver. From then on the cpufreq must be able
1050          * to accept all calls to ->verify and ->setpolicy for this CPU
1051          */
1052         ret = cpufreq_driver->init(policy);
1053         if (ret) {
1054                 pr_debug("initialization failed\n");
1055                 goto err_set_policy_cpu;
1056         }
1057
1058         /* related cpus should atleast have policy->cpus */
1059         cpumask_or(policy->related_cpus, policy->related_cpus, policy->cpus);
1060
1061         /*
1062          * affected cpus must always be the one, which are online. We aren't
1063          * managing offline cpus here.
1064          */
1065         cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
1066
1067         policy->user_policy.min = policy->min;
1068         policy->user_policy.max = policy->max;
1069
1070         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1071                                      CPUFREQ_START, policy);
1072
1073 #ifdef CONFIG_HOTPLUG_CPU
1074         gov = __find_governor(per_cpu(cpufreq_cpu_governor, cpu));
1075         if (gov) {
1076                 policy->governor = gov;
1077                 pr_debug("Restoring governor %s for cpu %d\n",
1078                        policy->governor->name, cpu);
1079         }
1080 #endif
1081
1082         write_lock_irqsave(&cpufreq_driver_lock, flags);
1083         for_each_cpu(j, policy->cpus) {
1084                 per_cpu(cpufreq_cpu_data, j) = policy;
1085                 per_cpu(cpufreq_policy_cpu, j) = policy->cpu;
1086         }
1087         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1088
1089         if (!frozen) {
1090                 ret = cpufreq_add_dev_interface(policy, dev);
1091                 if (ret)
1092                         goto err_out_unregister;
1093         }
1094
1095         cpufreq_init_policy(policy);
1096
1097         kobject_uevent(&policy->kobj, KOBJ_ADD);
1098         module_put(cpufreq_driver->owner);
1099         pr_debug("initialization complete\n");
1100
1101         return 0;
1102
1103 err_out_unregister:
1104         write_lock_irqsave(&cpufreq_driver_lock, flags);
1105         for_each_cpu(j, policy->cpus) {
1106                 per_cpu(cpufreq_cpu_data, j) = NULL;
1107                 if (j != cpu)
1108                         per_cpu(cpufreq_policy_cpu, j) = -1;
1109         }
1110         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1111
1112 err_set_policy_cpu:
1113         per_cpu(cpufreq_policy_cpu, cpu) = -1;
1114         cpufreq_policy_free(policy);
1115 nomem_out:
1116         module_put(cpufreq_driver->owner);
1117 module_out:
1118         return ret;
1119 }
1120
1121 /**
1122  * cpufreq_add_dev - add a CPU device
1123  *
1124  * Adds the cpufreq interface for a CPU device.
1125  *
1126  * The Oracle says: try running cpufreq registration/unregistration concurrently
1127  * with with cpu hotplugging and all hell will break loose. Tried to clean this
1128  * mess up, but more thorough testing is needed. - Mathieu
1129  */
1130 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1131 {
1132         return __cpufreq_add_dev(dev, sif, false);
1133 }
1134
1135 static void update_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu)
1136 {
1137         int j;
1138
1139         policy->last_cpu = policy->cpu;
1140         policy->cpu = cpu;
1141
1142         for_each_cpu(j, policy->cpus)
1143                 per_cpu(cpufreq_policy_cpu, j) = cpu;
1144
1145 #ifdef CONFIG_CPU_FREQ_TABLE
1146         cpufreq_frequency_table_update_policy_cpu(policy);
1147 #endif
1148         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1149                         CPUFREQ_UPDATE_POLICY_CPU, policy);
1150 }
1151
1152 static int cpufreq_nominate_new_policy_cpu(struct cpufreq_policy *data,
1153                                            unsigned int old_cpu, bool frozen)
1154 {
1155         struct device *cpu_dev;
1156         unsigned long flags;
1157         int ret;
1158
1159         /* first sibling now owns the new sysfs dir */
1160         cpu_dev = get_cpu_device(cpumask_first(data->cpus));
1161
1162         /* Don't touch sysfs files during light-weight tear-down */
1163         if (frozen)
1164                 return cpu_dev->id;
1165
1166         sysfs_remove_link(&cpu_dev->kobj, "cpufreq");
1167         ret = kobject_move(&data->kobj, &cpu_dev->kobj);
1168         if (ret) {
1169                 pr_err("%s: Failed to move kobj: %d", __func__, ret);
1170
1171                 WARN_ON(lock_policy_rwsem_write(old_cpu));
1172                 cpumask_set_cpu(old_cpu, data->cpus);
1173
1174                 write_lock_irqsave(&cpufreq_driver_lock, flags);
1175                 per_cpu(cpufreq_cpu_data, old_cpu) = data;
1176                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1177
1178                 unlock_policy_rwsem_write(old_cpu);
1179
1180                 ret = sysfs_create_link(&cpu_dev->kobj, &data->kobj,
1181                                         "cpufreq");
1182
1183                 return -EINVAL;
1184         }
1185
1186         return cpu_dev->id;
1187 }
1188
1189 /**
1190  * __cpufreq_remove_dev - remove a CPU device
1191  *
1192  * Removes the cpufreq interface for a CPU device.
1193  * Caller should already have policy_rwsem in write mode for this CPU.
1194  * This routine frees the rwsem before returning.
1195  */
1196 static int __cpufreq_remove_dev(struct device *dev,
1197                                 struct subsys_interface *sif, bool frozen)
1198 {
1199         unsigned int cpu = dev->id, cpus;
1200         int new_cpu;
1201         unsigned long flags;
1202         struct cpufreq_policy *data;
1203         struct kobject *kobj;
1204         struct completion *cmp;
1205
1206         pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
1207
1208         write_lock_irqsave(&cpufreq_driver_lock, flags);
1209
1210         data = per_cpu(cpufreq_cpu_data, cpu);
1211         per_cpu(cpufreq_cpu_data, cpu) = NULL;
1212
1213         /* Save the policy somewhere when doing a light-weight tear-down */
1214         if (frozen)
1215                 per_cpu(cpufreq_cpu_data_fallback, cpu) = data;
1216
1217         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1218
1219         if (!data) {
1220                 pr_debug("%s: No cpu_data found\n", __func__);
1221                 return -EINVAL;
1222         }
1223
1224         if (cpufreq_driver->target)
1225                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1226
1227 #ifdef CONFIG_HOTPLUG_CPU
1228         if (!cpufreq_driver->setpolicy)
1229                 strncpy(per_cpu(cpufreq_cpu_governor, cpu),
1230                         data->governor->name, CPUFREQ_NAME_LEN);
1231 #endif
1232
1233         WARN_ON(lock_policy_rwsem_write(cpu));
1234         cpus = cpumask_weight(data->cpus);
1235
1236         if (cpus > 1)
1237                 cpumask_clear_cpu(cpu, data->cpus);
1238         unlock_policy_rwsem_write(cpu);
1239
1240         if (cpu != data->cpu && !frozen) {
1241                 sysfs_remove_link(&dev->kobj, "cpufreq");
1242         } else if (cpus > 1) {
1243
1244                 new_cpu = cpufreq_nominate_new_policy_cpu(data, cpu, frozen);
1245                 if (new_cpu >= 0) {
1246                         WARN_ON(lock_policy_rwsem_write(cpu));
1247                         update_policy_cpu(data, new_cpu);
1248                         unlock_policy_rwsem_write(cpu);
1249
1250                         if (!frozen) {
1251                                 pr_debug("%s: policy Kobject moved to cpu: %d "
1252                                          "from: %d\n",__func__, new_cpu, cpu);
1253                         }
1254                 }
1255         }
1256
1257         /* If cpu is last user of policy, free policy */
1258         if (cpus == 1) {
1259                 if (cpufreq_driver->target)
1260                         __cpufreq_governor(data, CPUFREQ_GOV_POLICY_EXIT);
1261
1262                 if (!frozen) {
1263                         lock_policy_rwsem_read(cpu);
1264                         kobj = &data->kobj;
1265                         cmp = &data->kobj_unregister;
1266                         unlock_policy_rwsem_read(cpu);
1267                         kobject_put(kobj);
1268
1269                         /*
1270                          * We need to make sure that the underlying kobj is
1271                          * actually not referenced anymore by anybody before we
1272                          * proceed with unloading.
1273                          */
1274                         pr_debug("waiting for dropping of refcount\n");
1275                         wait_for_completion(cmp);
1276                         pr_debug("wait complete\n");
1277                 }
1278
1279                 /*
1280                  * Perform the ->exit() even during light-weight tear-down,
1281                  * since this is a core component, and is essential for the
1282                  * subsequent light-weight ->init() to succeed.
1283                  */
1284                 if (cpufreq_driver->exit)
1285                         cpufreq_driver->exit(data);
1286
1287                 if (!frozen)
1288                         cpufreq_policy_free(data);
1289         } else {
1290                 if (cpufreq_driver->target) {
1291                         __cpufreq_governor(data, CPUFREQ_GOV_START);
1292                         __cpufreq_governor(data, CPUFREQ_GOV_LIMITS);
1293                 }
1294         }
1295
1296         per_cpu(cpufreq_policy_cpu, cpu) = -1;
1297         return 0;
1298 }
1299
1300 static int cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1301 {
1302         unsigned int cpu = dev->id;
1303         int retval;
1304
1305         if (cpu_is_offline(cpu))
1306                 return 0;
1307
1308         retval = __cpufreq_remove_dev(dev, sif, false);
1309         return retval;
1310 }
1311
1312 static void handle_update(struct work_struct *work)
1313 {
1314         struct cpufreq_policy *policy =
1315                 container_of(work, struct cpufreq_policy, update);
1316         unsigned int cpu = policy->cpu;
1317         pr_debug("handle_update for cpu %u called\n", cpu);
1318         cpufreq_update_policy(cpu);
1319 }
1320
1321 /**
1322  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're
1323  *      in deep trouble.
1324  *      @cpu: cpu number
1325  *      @old_freq: CPU frequency the kernel thinks the CPU runs at
1326  *      @new_freq: CPU frequency the CPU actually runs at
1327  *
1328  *      We adjust to current frequency first, and need to clean up later.
1329  *      So either call to cpufreq_update_policy() or schedule handle_update()).
1330  */
1331 static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,
1332                                 unsigned int new_freq)
1333 {
1334         struct cpufreq_policy *policy;
1335         struct cpufreq_freqs freqs;
1336         unsigned long flags;
1337
1338         pr_debug("Warning: CPU frequency out of sync: cpufreq and timing "
1339                "core thinks of %u, is %u kHz.\n", old_freq, new_freq);
1340
1341         freqs.old = old_freq;
1342         freqs.new = new_freq;
1343
1344         read_lock_irqsave(&cpufreq_driver_lock, flags);
1345         policy = per_cpu(cpufreq_cpu_data, cpu);
1346         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1347
1348         cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE);
1349         cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE);
1350 }
1351
1352 /**
1353  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1354  * @cpu: CPU number
1355  *
1356  * This is the last known freq, without actually getting it from the driver.
1357  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1358  */
1359 unsigned int cpufreq_quick_get(unsigned int cpu)
1360 {
1361         struct cpufreq_policy *policy;
1362         unsigned int ret_freq = 0;
1363
1364         if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
1365                 return cpufreq_driver->get(cpu);
1366
1367         policy = cpufreq_cpu_get(cpu);
1368         if (policy) {
1369                 ret_freq = policy->cur;
1370                 cpufreq_cpu_put(policy);
1371         }
1372
1373         return ret_freq;
1374 }
1375 EXPORT_SYMBOL(cpufreq_quick_get);
1376
1377 /**
1378  * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1379  * @cpu: CPU number
1380  *
1381  * Just return the max possible frequency for a given CPU.
1382  */
1383 unsigned int cpufreq_quick_get_max(unsigned int cpu)
1384 {
1385         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1386         unsigned int ret_freq = 0;
1387
1388         if (policy) {
1389                 ret_freq = policy->max;
1390                 cpufreq_cpu_put(policy);
1391         }
1392
1393         return ret_freq;
1394 }
1395 EXPORT_SYMBOL(cpufreq_quick_get_max);
1396
1397 static unsigned int __cpufreq_get(unsigned int cpu)
1398 {
1399         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1400         unsigned int ret_freq = 0;
1401
1402         if (!cpufreq_driver->get)
1403                 return ret_freq;
1404
1405         ret_freq = cpufreq_driver->get(cpu);
1406
1407         if (ret_freq && policy->cur &&
1408                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1409                 /* verify no discrepancy between actual and
1410                                         saved value exists */
1411                 if (unlikely(ret_freq != policy->cur)) {
1412                         cpufreq_out_of_sync(cpu, policy->cur, ret_freq);
1413                         schedule_work(&policy->update);
1414                 }
1415         }
1416
1417         return ret_freq;
1418 }
1419
1420 /**
1421  * cpufreq_get - get the current CPU frequency (in kHz)
1422  * @cpu: CPU number
1423  *
1424  * Get the CPU current (static) CPU frequency
1425  */
1426 unsigned int cpufreq_get(unsigned int cpu)
1427 {
1428         unsigned int ret_freq = 0;
1429         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1430
1431         if (!policy)
1432                 goto out;
1433
1434         if (unlikely(lock_policy_rwsem_read(cpu)))
1435                 goto out_policy;
1436
1437         ret_freq = __cpufreq_get(cpu);
1438
1439         unlock_policy_rwsem_read(cpu);
1440
1441 out_policy:
1442         cpufreq_cpu_put(policy);
1443 out:
1444         return ret_freq;
1445 }
1446 EXPORT_SYMBOL(cpufreq_get);
1447
1448 static struct subsys_interface cpufreq_interface = {
1449         .name           = "cpufreq",
1450         .subsys         = &cpu_subsys,
1451         .add_dev        = cpufreq_add_dev,
1452         .remove_dev     = cpufreq_remove_dev,
1453 };
1454
1455 /**
1456  * cpufreq_bp_suspend - Prepare the boot CPU for system suspend.
1457  *
1458  * This function is only executed for the boot processor.  The other CPUs
1459  * have been put offline by means of CPU hotplug.
1460  */
1461 static int cpufreq_bp_suspend(void)
1462 {
1463         int ret = 0;
1464
1465         int cpu = smp_processor_id();
1466         struct cpufreq_policy *cpu_policy;
1467
1468         pr_debug("suspending cpu %u\n", cpu);
1469
1470         /* If there's no policy for the boot CPU, we have nothing to do. */
1471         cpu_policy = cpufreq_cpu_get(cpu);
1472         if (!cpu_policy)
1473                 return 0;
1474
1475         if (cpufreq_driver->suspend) {
1476                 ret = cpufreq_driver->suspend(cpu_policy);
1477                 if (ret)
1478                         printk(KERN_ERR "cpufreq: suspend failed in ->suspend "
1479                                         "step on CPU %u\n", cpu_policy->cpu);
1480         }
1481
1482         cpufreq_cpu_put(cpu_policy);
1483         return ret;
1484 }
1485
1486 /**
1487  * cpufreq_bp_resume - Restore proper frequency handling of the boot CPU.
1488  *
1489  *      1.) resume CPUfreq hardware support (cpufreq_driver->resume())
1490  *      2.) schedule call cpufreq_update_policy() ASAP as interrupts are
1491  *          restored. It will verify that the current freq is in sync with
1492  *          what we believe it to be. This is a bit later than when it
1493  *          should be, but nonethteless it's better than calling
1494  *          cpufreq_driver->get() here which might re-enable interrupts...
1495  *
1496  * This function is only executed for the boot CPU.  The other CPUs have not
1497  * been turned on yet.
1498  */
1499 static void cpufreq_bp_resume(void)
1500 {
1501         int ret = 0;
1502
1503         int cpu = smp_processor_id();
1504         struct cpufreq_policy *cpu_policy;
1505
1506         pr_debug("resuming cpu %u\n", cpu);
1507
1508         /* If there's no policy for the boot CPU, we have nothing to do. */
1509         cpu_policy = cpufreq_cpu_get(cpu);
1510         if (!cpu_policy)
1511                 return;
1512
1513         if (cpufreq_driver->resume) {
1514                 ret = cpufreq_driver->resume(cpu_policy);
1515                 if (ret) {
1516                         printk(KERN_ERR "cpufreq: resume failed in ->resume "
1517                                         "step on CPU %u\n", cpu_policy->cpu);
1518                         goto fail;
1519                 }
1520         }
1521
1522         schedule_work(&cpu_policy->update);
1523
1524 fail:
1525         cpufreq_cpu_put(cpu_policy);
1526 }
1527
1528 static struct syscore_ops cpufreq_syscore_ops = {
1529         .suspend        = cpufreq_bp_suspend,
1530         .resume         = cpufreq_bp_resume,
1531 };
1532
1533 /**
1534  *      cpufreq_get_current_driver - return current driver's name
1535  *
1536  *      Return the name string of the currently loaded cpufreq driver
1537  *      or NULL, if none.
1538  */
1539 const char *cpufreq_get_current_driver(void)
1540 {
1541         if (cpufreq_driver)
1542                 return cpufreq_driver->name;
1543
1544         return NULL;
1545 }
1546 EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
1547
1548 /*********************************************************************
1549  *                     NOTIFIER LISTS INTERFACE                      *
1550  *********************************************************************/
1551
1552 /**
1553  *      cpufreq_register_notifier - register a driver with cpufreq
1554  *      @nb: notifier function to register
1555  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1556  *
1557  *      Add a driver to one of two lists: either a list of drivers that
1558  *      are notified about clock rate changes (once before and once after
1559  *      the transition), or a list of drivers that are notified about
1560  *      changes in cpufreq policy.
1561  *
1562  *      This function may sleep, and has the same return conditions as
1563  *      blocking_notifier_chain_register.
1564  */
1565 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1566 {
1567         int ret;
1568
1569         if (cpufreq_disabled())
1570                 return -EINVAL;
1571
1572         WARN_ON(!init_cpufreq_transition_notifier_list_called);
1573
1574         switch (list) {
1575         case CPUFREQ_TRANSITION_NOTIFIER:
1576                 ret = srcu_notifier_chain_register(
1577                                 &cpufreq_transition_notifier_list, nb);
1578                 break;
1579         case CPUFREQ_POLICY_NOTIFIER:
1580                 ret = blocking_notifier_chain_register(
1581                                 &cpufreq_policy_notifier_list, nb);
1582                 break;
1583         default:
1584                 ret = -EINVAL;
1585         }
1586
1587         return ret;
1588 }
1589 EXPORT_SYMBOL(cpufreq_register_notifier);
1590
1591 /**
1592  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1593  *      @nb: notifier block to be unregistered
1594  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1595  *
1596  *      Remove a driver from the CPU frequency notifier list.
1597  *
1598  *      This function may sleep, and has the same return conditions as
1599  *      blocking_notifier_chain_unregister.
1600  */
1601 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1602 {
1603         int ret;
1604
1605         if (cpufreq_disabled())
1606                 return -EINVAL;
1607
1608         switch (list) {
1609         case CPUFREQ_TRANSITION_NOTIFIER:
1610                 ret = srcu_notifier_chain_unregister(
1611                                 &cpufreq_transition_notifier_list, nb);
1612                 break;
1613         case CPUFREQ_POLICY_NOTIFIER:
1614                 ret = blocking_notifier_chain_unregister(
1615                                 &cpufreq_policy_notifier_list, nb);
1616                 break;
1617         default:
1618                 ret = -EINVAL;
1619         }
1620
1621         return ret;
1622 }
1623 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1624
1625
1626 /*********************************************************************
1627  *                              GOVERNORS                            *
1628  *********************************************************************/
1629
1630 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1631                             unsigned int target_freq,
1632                             unsigned int relation)
1633 {
1634         int retval = -EINVAL;
1635         unsigned int old_target_freq = target_freq;
1636
1637         if (cpufreq_disabled())
1638                 return -ENODEV;
1639         if (policy->transition_ongoing)
1640                 return -EBUSY;
1641
1642         /* Make sure that target_freq is within supported range */
1643         if (target_freq > policy->max)
1644                 target_freq = policy->max;
1645         if (target_freq < policy->min)
1646                 target_freq = policy->min;
1647
1648         pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
1649                         policy->cpu, target_freq, relation, old_target_freq);
1650
1651         if (target_freq == policy->cur)
1652                 return 0;
1653
1654         if (cpufreq_driver->target)
1655                 retval = cpufreq_driver->target(policy, target_freq, relation);
1656
1657         return retval;
1658 }
1659 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1660
1661 int cpufreq_driver_target(struct cpufreq_policy *policy,
1662                           unsigned int target_freq,
1663                           unsigned int relation)
1664 {
1665         int ret = -EINVAL;
1666
1667         if (unlikely(lock_policy_rwsem_write(policy->cpu)))
1668                 goto fail;
1669
1670         ret = __cpufreq_driver_target(policy, target_freq, relation);
1671
1672         unlock_policy_rwsem_write(policy->cpu);
1673
1674 fail:
1675         return ret;
1676 }
1677 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
1678
1679 int __cpufreq_driver_getavg(struct cpufreq_policy *policy, unsigned int cpu)
1680 {
1681         if (cpufreq_disabled())
1682                 return 0;
1683
1684         if (!cpufreq_driver->getavg)
1685                 return 0;
1686
1687         return cpufreq_driver->getavg(policy, cpu);
1688 }
1689 EXPORT_SYMBOL_GPL(__cpufreq_driver_getavg);
1690
1691 /*
1692  * when "event" is CPUFREQ_GOV_LIMITS
1693  */
1694
1695 static int __cpufreq_governor(struct cpufreq_policy *policy,
1696                                         unsigned int event)
1697 {
1698         int ret;
1699
1700         /* Only must be defined when default governor is known to have latency
1701            restrictions, like e.g. conservative or ondemand.
1702            That this is the case is already ensured in Kconfig
1703         */
1704 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
1705         struct cpufreq_governor *gov = &cpufreq_gov_performance;
1706 #else
1707         struct cpufreq_governor *gov = NULL;
1708 #endif
1709
1710         if (policy->governor->max_transition_latency &&
1711             policy->cpuinfo.transition_latency >
1712             policy->governor->max_transition_latency) {
1713                 if (!gov)
1714                         return -EINVAL;
1715                 else {
1716                         printk(KERN_WARNING "%s governor failed, too long"
1717                                " transition latency of HW, fallback"
1718                                " to %s governor\n",
1719                                policy->governor->name,
1720                                gov->name);
1721                         policy->governor = gov;
1722                 }
1723         }
1724
1725         if (!try_module_get(policy->governor->owner))
1726                 return -EINVAL;
1727
1728         pr_debug("__cpufreq_governor for CPU %u, event %u\n",
1729                                                 policy->cpu, event);
1730
1731         mutex_lock(&cpufreq_governor_lock);
1732         if ((!policy->governor_enabled && (event == CPUFREQ_GOV_STOP)) ||
1733             (policy->governor_enabled && (event == CPUFREQ_GOV_START))) {
1734                 mutex_unlock(&cpufreq_governor_lock);
1735                 return -EBUSY;
1736         }
1737
1738         if (event == CPUFREQ_GOV_STOP)
1739                 policy->governor_enabled = false;
1740         else if (event == CPUFREQ_GOV_START)
1741                 policy->governor_enabled = true;
1742
1743         mutex_unlock(&cpufreq_governor_lock);
1744
1745         ret = policy->governor->governor(policy, event);
1746
1747         if (!ret) {
1748                 if (event == CPUFREQ_GOV_POLICY_INIT)
1749                         policy->governor->initialized++;
1750                 else if (event == CPUFREQ_GOV_POLICY_EXIT)
1751                         policy->governor->initialized--;
1752         } else {
1753                 /* Restore original values */
1754                 mutex_lock(&cpufreq_governor_lock);
1755                 if (event == CPUFREQ_GOV_STOP)
1756                         policy->governor_enabled = true;
1757                 else if (event == CPUFREQ_GOV_START)
1758                         policy->governor_enabled = false;
1759                 mutex_unlock(&cpufreq_governor_lock);
1760         }
1761
1762         /* we keep one module reference alive for
1763                         each CPU governed by this CPU */
1764         if ((event != CPUFREQ_GOV_START) || ret)
1765                 module_put(policy->governor->owner);
1766         if ((event == CPUFREQ_GOV_STOP) && !ret)
1767                 module_put(policy->governor->owner);
1768
1769         return ret;
1770 }
1771
1772 int cpufreq_register_governor(struct cpufreq_governor *governor)
1773 {
1774         int err;
1775
1776         if (!governor)
1777                 return -EINVAL;
1778
1779         if (cpufreq_disabled())
1780                 return -ENODEV;
1781
1782         mutex_lock(&cpufreq_governor_mutex);
1783
1784         governor->initialized = 0;
1785         err = -EBUSY;
1786         if (__find_governor(governor->name) == NULL) {
1787                 err = 0;
1788                 list_add(&governor->governor_list, &cpufreq_governor_list);
1789         }
1790
1791         mutex_unlock(&cpufreq_governor_mutex);
1792         return err;
1793 }
1794 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
1795
1796 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
1797 {
1798 #ifdef CONFIG_HOTPLUG_CPU
1799         int cpu;
1800 #endif
1801
1802         if (!governor)
1803                 return;
1804
1805         if (cpufreq_disabled())
1806                 return;
1807
1808 #ifdef CONFIG_HOTPLUG_CPU
1809         for_each_present_cpu(cpu) {
1810                 if (cpu_online(cpu))
1811                         continue;
1812                 if (!strcmp(per_cpu(cpufreq_cpu_governor, cpu), governor->name))
1813                         strcpy(per_cpu(cpufreq_cpu_governor, cpu), "\0");
1814         }
1815 #endif
1816
1817         mutex_lock(&cpufreq_governor_mutex);
1818         list_del(&governor->governor_list);
1819         mutex_unlock(&cpufreq_governor_mutex);
1820         return;
1821 }
1822 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
1823
1824
1825 /*********************************************************************
1826  *                          POLICY INTERFACE                         *
1827  *********************************************************************/
1828
1829 /**
1830  * cpufreq_get_policy - get the current cpufreq_policy
1831  * @policy: struct cpufreq_policy into which the current cpufreq_policy
1832  *      is written
1833  *
1834  * Reads the current cpufreq policy.
1835  */
1836 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
1837 {
1838         struct cpufreq_policy *cpu_policy;
1839         if (!policy)
1840                 return -EINVAL;
1841
1842         cpu_policy = cpufreq_cpu_get(cpu);
1843         if (!cpu_policy)
1844                 return -EINVAL;
1845
1846         memcpy(policy, cpu_policy, sizeof(struct cpufreq_policy));
1847
1848         cpufreq_cpu_put(cpu_policy);
1849         return 0;
1850 }
1851 EXPORT_SYMBOL(cpufreq_get_policy);
1852
1853 /*
1854  * data   : current policy.
1855  * policy : policy to be set.
1856  */
1857 static int __cpufreq_set_policy(struct cpufreq_policy *data,
1858                                 struct cpufreq_policy *policy)
1859 {
1860         int ret = 0, failed = 1;
1861
1862         pr_debug("setting new policy for CPU %u: %u - %u kHz\n", policy->cpu,
1863                 policy->min, policy->max);
1864
1865         memcpy(&policy->cpuinfo, &data->cpuinfo,
1866                                 sizeof(struct cpufreq_cpuinfo));
1867
1868         if (policy->min > data->max || policy->max < data->min) {
1869                 ret = -EINVAL;
1870                 goto error_out;
1871         }
1872
1873         /* verify the cpu speed can be set within this limit */
1874         ret = cpufreq_driver->verify(policy);
1875         if (ret)
1876                 goto error_out;
1877
1878         /* adjust if necessary - all reasons */
1879         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1880                         CPUFREQ_ADJUST, policy);
1881
1882         /* adjust if necessary - hardware incompatibility*/
1883         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1884                         CPUFREQ_INCOMPATIBLE, policy);
1885
1886         /*
1887          * verify the cpu speed can be set within this limit, which might be
1888          * different to the first one
1889          */
1890         ret = cpufreq_driver->verify(policy);
1891         if (ret)
1892                 goto error_out;
1893
1894         /* notification of the new policy */
1895         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1896                         CPUFREQ_NOTIFY, policy);
1897
1898         data->min = policy->min;
1899         data->max = policy->max;
1900
1901         pr_debug("new min and max freqs are %u - %u kHz\n",
1902                                         data->min, data->max);
1903
1904         if (cpufreq_driver->setpolicy) {
1905                 data->policy = policy->policy;
1906                 pr_debug("setting range\n");
1907                 ret = cpufreq_driver->setpolicy(policy);
1908         } else {
1909                 if (policy->governor != data->governor) {
1910                         /* save old, working values */
1911                         struct cpufreq_governor *old_gov = data->governor;
1912
1913                         pr_debug("governor switch\n");
1914
1915                         /* end old governor */
1916                         if (data->governor) {
1917                                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1918                                 unlock_policy_rwsem_write(policy->cpu);
1919                                 __cpufreq_governor(data,
1920                                                 CPUFREQ_GOV_POLICY_EXIT);
1921                                 lock_policy_rwsem_write(policy->cpu);
1922                         }
1923
1924                         /* start new governor */
1925                         data->governor = policy->governor;
1926                         if (!__cpufreq_governor(data, CPUFREQ_GOV_POLICY_INIT)) {
1927                                 if (!__cpufreq_governor(data, CPUFREQ_GOV_START)) {
1928                                         failed = 0;
1929                                 } else {
1930                                         unlock_policy_rwsem_write(policy->cpu);
1931                                         __cpufreq_governor(data,
1932                                                         CPUFREQ_GOV_POLICY_EXIT);
1933                                         lock_policy_rwsem_write(policy->cpu);
1934                                 }
1935                         }
1936
1937                         if (failed) {
1938                                 /* new governor failed, so re-start old one */
1939                                 pr_debug("starting governor %s failed\n",
1940                                                         data->governor->name);
1941                                 if (old_gov) {
1942                                         data->governor = old_gov;
1943                                         __cpufreq_governor(data,
1944                                                         CPUFREQ_GOV_POLICY_INIT);
1945                                         __cpufreq_governor(data,
1946                                                            CPUFREQ_GOV_START);
1947                                 }
1948                                 ret = -EINVAL;
1949                                 goto error_out;
1950                         }
1951                         /* might be a policy change, too, so fall through */
1952                 }
1953                 pr_debug("governor: change or update limits\n");
1954                 __cpufreq_governor(data, CPUFREQ_GOV_LIMITS);
1955         }
1956
1957 error_out:
1958         return ret;
1959 }
1960
1961 /**
1962  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
1963  *      @cpu: CPU which shall be re-evaluated
1964  *
1965  *      Useful for policy notifiers which have different necessities
1966  *      at different times.
1967  */
1968 int cpufreq_update_policy(unsigned int cpu)
1969 {
1970         struct cpufreq_policy *data = cpufreq_cpu_get(cpu);
1971         struct cpufreq_policy policy;
1972         int ret;
1973
1974         if (!data) {
1975                 ret = -ENODEV;
1976                 goto no_policy;
1977         }
1978
1979         if (unlikely(lock_policy_rwsem_write(cpu))) {
1980                 ret = -EINVAL;
1981                 goto fail;
1982         }
1983
1984         pr_debug("updating policy for CPU %u\n", cpu);
1985         memcpy(&policy, data, sizeof(struct cpufreq_policy));
1986         policy.min = data->user_policy.min;
1987         policy.max = data->user_policy.max;
1988         policy.policy = data->user_policy.policy;
1989         policy.governor = data->user_policy.governor;
1990
1991         /*
1992          * BIOS might change freq behind our back
1993          * -> ask driver for current freq and notify governors about a change
1994          */
1995         if (cpufreq_driver->get) {
1996                 policy.cur = cpufreq_driver->get(cpu);
1997                 if (!data->cur) {
1998                         pr_debug("Driver did not initialize current freq");
1999                         data->cur = policy.cur;
2000                 } else {
2001                         if (data->cur != policy.cur && cpufreq_driver->target)
2002                                 cpufreq_out_of_sync(cpu, data->cur,
2003                                                                 policy.cur);
2004                 }
2005         }
2006
2007         ret = __cpufreq_set_policy(data, &policy);
2008
2009         unlock_policy_rwsem_write(cpu);
2010
2011 fail:
2012         cpufreq_cpu_put(data);
2013 no_policy:
2014         return ret;
2015 }
2016 EXPORT_SYMBOL(cpufreq_update_policy);
2017
2018 static int cpufreq_cpu_callback(struct notifier_block *nfb,
2019                                         unsigned long action, void *hcpu)
2020 {
2021         unsigned int cpu = (unsigned long)hcpu;
2022         struct device *dev;
2023         bool frozen = false;
2024
2025         dev = get_cpu_device(cpu);
2026         if (dev) {
2027
2028                 if (action & CPU_TASKS_FROZEN)
2029                         frozen = true;
2030
2031                 switch (action & ~CPU_TASKS_FROZEN) {
2032                 case CPU_ONLINE:
2033                         __cpufreq_add_dev(dev, NULL, frozen);
2034                         cpufreq_update_policy(cpu);
2035                         break;
2036
2037                 case CPU_DOWN_PREPARE:
2038                         __cpufreq_remove_dev(dev, NULL, frozen);
2039                         break;
2040
2041                 case CPU_DOWN_FAILED:
2042                         __cpufreq_add_dev(dev, NULL, frozen);
2043                         break;
2044                 }
2045         }
2046         return NOTIFY_OK;
2047 }
2048
2049 static struct notifier_block __refdata cpufreq_cpu_notifier = {
2050         .notifier_call = cpufreq_cpu_callback,
2051 };
2052
2053 /*********************************************************************
2054  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
2055  *********************************************************************/
2056
2057 /**
2058  * cpufreq_register_driver - register a CPU Frequency driver
2059  * @driver_data: A struct cpufreq_driver containing the values#
2060  * submitted by the CPU Frequency driver.
2061  *
2062  * Registers a CPU Frequency driver to this core code. This code
2063  * returns zero on success, -EBUSY when another driver got here first
2064  * (and isn't unregistered in the meantime).
2065  *
2066  */
2067 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
2068 {
2069         unsigned long flags;
2070         int ret;
2071
2072         if (cpufreq_disabled())
2073                 return -ENODEV;
2074
2075         if (!driver_data || !driver_data->verify || !driver_data->init ||
2076             ((!driver_data->setpolicy) && (!driver_data->target)))
2077                 return -EINVAL;
2078
2079         pr_debug("trying to register driver %s\n", driver_data->name);
2080
2081         if (driver_data->setpolicy)
2082                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
2083
2084         write_lock_irqsave(&cpufreq_driver_lock, flags);
2085         if (cpufreq_driver) {
2086                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2087                 return -EBUSY;
2088         }
2089         cpufreq_driver = driver_data;
2090         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2091
2092         ret = subsys_interface_register(&cpufreq_interface);
2093         if (ret)
2094                 goto err_null_driver;
2095
2096         if (!(cpufreq_driver->flags & CPUFREQ_STICKY)) {
2097                 int i;
2098                 ret = -ENODEV;
2099
2100                 /* check for at least one working CPU */
2101                 for (i = 0; i < nr_cpu_ids; i++)
2102                         if (cpu_possible(i) && per_cpu(cpufreq_cpu_data, i)) {
2103                                 ret = 0;
2104                                 break;
2105                         }
2106
2107                 /* if all ->init() calls failed, unregister */
2108                 if (ret) {
2109                         pr_debug("no CPU initialized for driver %s\n",
2110                                                         driver_data->name);
2111                         goto err_if_unreg;
2112                 }
2113         }
2114
2115         register_hotcpu_notifier(&cpufreq_cpu_notifier);
2116         pr_debug("driver %s up and running\n", driver_data->name);
2117
2118         return 0;
2119 err_if_unreg:
2120         subsys_interface_unregister(&cpufreq_interface);
2121 err_null_driver:
2122         write_lock_irqsave(&cpufreq_driver_lock, flags);
2123         cpufreq_driver = NULL;
2124         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2125         return ret;
2126 }
2127 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
2128
2129 /**
2130  * cpufreq_unregister_driver - unregister the current CPUFreq driver
2131  *
2132  * Unregister the current CPUFreq driver. Only call this if you have
2133  * the right to do so, i.e. if you have succeeded in initialising before!
2134  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
2135  * currently not initialised.
2136  */
2137 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
2138 {
2139         unsigned long flags;
2140
2141         if (!cpufreq_driver || (driver != cpufreq_driver))
2142                 return -EINVAL;
2143
2144         pr_debug("unregistering driver %s\n", driver->name);
2145
2146         subsys_interface_unregister(&cpufreq_interface);
2147         unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
2148
2149         write_lock_irqsave(&cpufreq_driver_lock, flags);
2150         cpufreq_driver = NULL;
2151         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2152
2153         return 0;
2154 }
2155 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
2156
2157 static int __init cpufreq_core_init(void)
2158 {
2159         int cpu;
2160
2161         if (cpufreq_disabled())
2162                 return -ENODEV;
2163
2164         for_each_possible_cpu(cpu) {
2165                 per_cpu(cpufreq_policy_cpu, cpu) = -1;
2166                 init_rwsem(&per_cpu(cpu_policy_rwsem, cpu));
2167         }
2168
2169         cpufreq_global_kobject = kobject_create();
2170         BUG_ON(!cpufreq_global_kobject);
2171         register_syscore_ops(&cpufreq_syscore_ops);
2172
2173         return 0;
2174 }
2175 core_initcall(cpufreq_core_init);