drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6  *            (C) 2013 Viresh Kumar <viresh.kumar@linaro.org>
   7  *
   8  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   9  *      Added handling for CPU hotplug
  10  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  11  *      Fix handling for CPU hotplug -- affected CPUs
  12  *
  13  * This program is free software; you can redistribute it and/or modify
  14  * it under the terms of the GNU General Public License version 2 as
  15  * published by the Free Software Foundation.
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <asm/cputime.h>
  21 #include <linux/kernel.h>
  22 #include <linux/kernel_stat.h>
  23 #include <linux/module.h>
  24 #include <linux/init.h>
  25 #include <linux/notifier.h>
  26 #include <linux/cpufreq.h>
  27 #include <linux/delay.h>
  28 #include <linux/interrupt.h>
  29 #include <linux/spinlock.h>
  30 #include <linux/tick.h>
  31 #include <linux/device.h>
  32 #include <linux/slab.h>
  33 #include <linux/cpu.h>
  34 #include <linux/completion.h>
  35 #include <linux/mutex.h>
  36 #include <linux/syscore_ops.h>
  37
  38 #include <trace/events/power.h>
  39
  40 /**
  41  * The "cpufreq driver" - the arch- or hardware-dependent low
  42  * level driver of CPUFreq support, and its spinlock. This lock
  43  * also protects the cpufreq_cpu_data array.
  44  */
  45 static struct cpufreq_driver *cpufreq_driver;
  46 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
  47 static DEFINE_RWLOCK(cpufreq_driver_lock);
  48 static DEFINE_MUTEX(cpufreq_governor_lock);
  49
  50 #ifdef CONFIG_HOTPLUG_CPU
  51 /* This one keeps track of the previously set governor of a removed CPU */
  52 static DEFINE_PER_CPU(char[CPUFREQ_NAME_LEN], cpufreq_cpu_governor);
  53 #endif
  54
  55 /*
  56  * cpu_policy_rwsem is a per CPU reader-writer semaphore designed to cure
  57  * all cpufreq/hotplug/workqueue/etc related lock issues.
  58  *
  59  * The rules for this semaphore:
  60  * - Any routine that wants to read from the policy structure will
  61  *   do a down_read on this semaphore.
  62  * - Any routine that will write to the policy structure and/or may take away
  63  *   the policy altogether (eg. CPU hotplug), will hold this lock in write
  64  *   mode before doing so.
  65  *
  66  * Additional rules:
  67  * - Governor routines that can be called in cpufreq hotplug path should not
  68  *   take this sem as top level hotplug notifier handler takes this.
  69  * - Lock should not be held across
  70  *     __cpufreq_governor(data, CPUFREQ_GOV_STOP);
  71  */
  72 static DEFINE_PER_CPU(int, cpufreq_policy_cpu);
  73 static DEFINE_PER_CPU(struct rw_semaphore, cpu_policy_rwsem);
  74
  75 #define lock_policy_rwsem(mode, cpu)                                    \
  76 static int lock_policy_rwsem_##mode(int cpu)                            \
  77 {                                                                       \
  78         int policy_cpu = per_cpu(cpufreq_policy_cpu, cpu);              \
  79         BUG_ON(policy_cpu == -1);                                       \
  80         down_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));            \
  81                                                                         \
  82         return 0;                                                       \
  83 }
  84
  85 lock_policy_rwsem(read, cpu);
  86 lock_policy_rwsem(write, cpu);
  87
  88 #define unlock_policy_rwsem(mode, cpu)                                  \
  89 static void unlock_policy_rwsem_##mode(int cpu)                         \
  90 {                                                                       \
  91         int policy_cpu = per_cpu(cpufreq_policy_cpu, cpu);              \
  92         BUG_ON(policy_cpu == -1);                                       \
  93         up_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));              \
  94 }
  95
  96 unlock_policy_rwsem(read, cpu);
  97 unlock_policy_rwsem(write, cpu);
  98
  99 /* internal prototypes */
 100 static int __cpufreq_governor(struct cpufreq_policy *policy,
 101                 unsigned int event);
 102 static unsigned int __cpufreq_get(unsigned int cpu);
 103 static void handle_update(struct work_struct *work);
 104
 105 /**
 106  * Two notifier lists: the "policy" list is involved in the
 107  * validation process for a new CPU frequency policy; the
 108  * "transition" list for kernel code that needs to handle
 109  * changes to devices when the CPU clock speed changes.
 110  * The mutex locks both lists.
 111  */
 112 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
 113 static struct srcu_notifier_head cpufreq_transition_notifier_list;
 114
 115 static bool init_cpufreq_transition_notifier_list_called;
 116 static int __init init_cpufreq_transition_notifier_list(void)
 117 {
 118         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
 119         init_cpufreq_transition_notifier_list_called = true;
 120         return 0;
 121 }
 122 pure_initcall(init_cpufreq_transition_notifier_list);
 123
 124 static int off __read_mostly;
 125 static int cpufreq_disabled(void)
 126 {
 127         return off;
 128 }
 129 void disable_cpufreq(void)
 130 {
 131         off = 1;
 132 }
 133 static LIST_HEAD(cpufreq_governor_list);
 134 static DEFINE_MUTEX(cpufreq_governor_mutex);
 135
 136 bool have_governor_per_policy(void)
 137 {
 138         return cpufreq_driver->have_governor_per_policy;
 139 }
 140 EXPORT_SYMBOL_GPL(have_governor_per_policy);
 141
 142 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
 143 {
 144         if (have_governor_per_policy())
 145                 return &policy->kobj;
 146         else
 147                 return cpufreq_global_kobject;
 148 }
 149 EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
 150
 151 static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
 152 {
 153         u64 idle_time;
 154         u64 cur_wall_time;
 155         u64 busy_time;
 156
 157         cur_wall_time = jiffies64_to_cputime64(get_jiffies_64());
 158
 159         busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER];
 160         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM];
 161         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ];
 162         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ];
 163         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL];
 164         busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE];
 165
 166         idle_time = cur_wall_time - busy_time;
 167         if (wall)
 168                 *wall = cputime_to_usecs(cur_wall_time);
 169
 170         return cputime_to_usecs(idle_time);
 171 }
 172
 173 u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
 174 {
 175         u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);
 176
 177         if (idle_time == -1ULL)
 178                 return get_cpu_idle_time_jiffy(cpu, wall);
 179         else if (!io_busy)
 180                 idle_time += get_cpu_iowait_time_us(cpu, wall);
 181
 182         return idle_time;
 183 }
 184 EXPORT_SYMBOL_GPL(get_cpu_idle_time);
 185
 186 static struct cpufreq_policy *__cpufreq_cpu_get(unsigned int cpu, bool sysfs)
 187 {
 188         struct cpufreq_policy *data;
 189         unsigned long flags;
 190
 191         if (cpu >= nr_cpu_ids)
 192                 goto err_out;
 193
 194         /* get the cpufreq driver */
 195         read_lock_irqsave(&cpufreq_driver_lock, flags);
 196
 197         if (!cpufreq_driver)
 198                 goto err_out_unlock;
 199
 200         if (!try_module_get(cpufreq_driver->owner))
 201                 goto err_out_unlock;
 202
 203         /* get the CPU */
 204         data = per_cpu(cpufreq_cpu_data, cpu);
 205
 206         if (!data)
 207                 goto err_out_put_module;
 208
 209         if (!sysfs && !kobject_get(&data->kobj))
 210                 goto err_out_put_module;
 211
 212         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 213         return data;
 214
 215 err_out_put_module:
 216         module_put(cpufreq_driver->owner);
 217 err_out_unlock:
 218         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
 219 err_out:
 220         return NULL;
 221 }
 222
 223 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 224 {
 225         if (cpufreq_disabled())
 226                 return NULL;
 227
 228         return __cpufreq_cpu_get(cpu, false);
 229 }
 230 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 231
 232 static struct cpufreq_policy *cpufreq_cpu_get_sysfs(unsigned int cpu)
 233 {
 234         return __cpufreq_cpu_get(cpu, true);
 235 }
 236
 237 static void __cpufreq_cpu_put(struct cpufreq_policy *data, bool sysfs)
 238 {
 239         if (!sysfs)
 240                 kobject_put(&data->kobj);
 241         module_put(cpufreq_driver->owner);
 242 }
 243
 244 void cpufreq_cpu_put(struct cpufreq_policy *data)
 245 {
 246         if (cpufreq_disabled())
 247                 return;
 248
 249         __cpufreq_cpu_put(data, false);
 250 }
 251 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 252
 253 static void cpufreq_cpu_put_sysfs(struct cpufreq_policy *data)
 254 {
 255         __cpufreq_cpu_put(data, true);
 256 }
 257
 258 /*********************************************************************
 259  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 260  *********************************************************************/
 261
 262 /**
 263  * adjust_jiffies - adjust the system "loops_per_jiffy"
 264  *
 265  * This function alters the system "loops_per_jiffy" for the clock
 266  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 267  * systems as each CPU might be scaled differently. So, use the arch
 268  * per-CPU loops_per_jiffy value wherever possible.
 269  */
 270 #ifndef CONFIG_SMP
 271 static unsigned long l_p_j_ref;
 272 static unsigned int l_p_j_ref_freq;
 273
 274 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 275 {
 276         if (ci->flags & CPUFREQ_CONST_LOOPS)
 277                 return;
 278
 279         if (!l_p_j_ref_freq) {
 280                 l_p_j_ref = loops_per_jiffy;
 281                 l_p_j_ref_freq = ci->old;
 282                 pr_debug("saving %lu as reference value for loops_per_jiffy; "
 283                         "freq is %u kHz\n", l_p_j_ref, l_p_j_ref_freq);
 284         }
 285         if ((val == CPUFREQ_POSTCHANGE && ci->old != ci->new) ||
 286             (val == CPUFREQ_RESUMECHANGE || val == CPUFREQ_SUSPENDCHANGE)) {
 287                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 288                                                                 ci->new);
 289                 pr_debug("scaling loops_per_jiffy to %lu "
 290                         "for frequency %u kHz\n", loops_per_jiffy, ci->new);
 291         }
 292 }
 293 #else
 294 static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 295 {
 296         return;
 297 }
 298 #endif
 299
 300 static void __cpufreq_notify_transition(struct cpufreq_policy *policy,
 301                 struct cpufreq_freqs *freqs, unsigned int state)
 302 {
 303         BUG_ON(irqs_disabled());
 304
 305         if (cpufreq_disabled())
 306                 return;
 307
 308         freqs->flags = cpufreq_driver->flags;
 309         pr_debug("notification %u of frequency transition to %u kHz\n",
 310                 state, freqs->new);
 311
 312         switch (state) {
 313
 314         case CPUFREQ_PRECHANGE:
 315                 if (WARN(policy->transition_ongoing ==
 316                                         cpumask_weight(policy->cpus),
 317                                 "In middle of another frequency transition\n"))
 318                         return;
 319
 320                 policy->transition_ongoing++;
 321
 322                 /* detect if the driver reported a value as "old frequency"
 323                  * which is not equal to what the cpufreq core thinks is
 324                  * "old frequency".
 325                  */
 326                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 327                         if ((policy) && (policy->cpu == freqs->cpu) &&
 328                             (policy->cur) && (policy->cur != freqs->old)) {
 329                                 pr_debug("Warning: CPU frequency is"
 330                                         " %u, cpufreq assumed %u kHz.\n",
 331                                         freqs->old, policy->cur);
 332                                 freqs->old = policy->cur;
 333                         }
 334                 }
 335                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 336                                 CPUFREQ_PRECHANGE, freqs);
 337                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 338                 break;
 339
 340         case CPUFREQ_POSTCHANGE:
 341                 if (WARN(!policy->transition_ongoing,
 342                                 "No frequency transition in progress\n"))
 343                         return;
 344
 345                 policy->transition_ongoing--;
 346
 347                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 348                 pr_debug("FREQ: %lu - CPU: %lu", (unsigned long)freqs->new,
 349                         (unsigned long)freqs->cpu);
 350                 trace_cpu_frequency(freqs->new, freqs->cpu);
 351                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 352                                 CPUFREQ_POSTCHANGE, freqs);
 353                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 354                         policy->cur = freqs->new;
 355                 break;
 356         }
 357 }
 358
 359 /**
 360  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 361  * on frequency transition.
 362  *
 363  * This function calls the transition notifiers and the "adjust_jiffies"
 364  * function. It is called twice on all CPU frequency changes that have
 365  * external effects.
 366  */
 367 void cpufreq_notify_transition(struct cpufreq_policy *policy,
 368                 struct cpufreq_freqs *freqs, unsigned int state)
 369 {
 370         for_each_cpu(freqs->cpu, policy->cpus)
 371                 __cpufreq_notify_transition(policy, freqs, state);
 372 }
 373 EXPORT_SYMBOL_GPL(cpufreq_notify_transition);
 374
 375
 376 /*********************************************************************
 377  *                          SYSFS INTERFACE                          *
 378  *********************************************************************/
 379
 380 static struct cpufreq_governor *__find_governor(const char *str_governor)
 381 {
 382         struct cpufreq_governor *t;
 383
 384         list_for_each_entry(t, &cpufreq_governor_list, governor_list)
 385                 if (!strnicmp(str_governor, t->name, CPUFREQ_NAME_LEN))
 386                         return t;
 387
 388         return NULL;
 389 }
 390
 391 /**
 392  * cpufreq_parse_governor - parse a governor string
 393  */
 394 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
 395                                 struct cpufreq_governor **governor)
 396 {
 397         int err = -EINVAL;
 398
 399         if (!cpufreq_driver)
 400                 goto out;
 401
 402         if (cpufreq_driver->setpolicy) {
 403                 if (!strnicmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 404                         *policy = CPUFREQ_POLICY_PERFORMANCE;
 405                         err = 0;
 406                 } else if (!strnicmp(str_governor, "powersave",
 407                                                 CPUFREQ_NAME_LEN)) {
 408                         *policy = CPUFREQ_POLICY_POWERSAVE;
 409                         err = 0;
 410                 }
 411         } else if (cpufreq_driver->target) {
 412                 struct cpufreq_governor *t;
 413
 414                 mutex_lock(&cpufreq_governor_mutex);
 415
 416                 t = __find_governor(str_governor);
 417
 418                 if (t == NULL) {
 419                         int ret;
 420
 421                         mutex_unlock(&cpufreq_governor_mutex);
 422                         ret = request_module("cpufreq_%s", str_governor);
 423                         mutex_lock(&cpufreq_governor_mutex);
 424
 425                         if (ret == 0)
 426                                 t = __find_governor(str_governor);
 427                 }
 428
 429                 if (t != NULL) {
 430                         *governor = t;
 431                         err = 0;
 432                 }
 433
 434                 mutex_unlock(&cpufreq_governor_mutex);
 435         }
 436 out:
 437         return err;
 438 }
 439
 440 /**
 441  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 442  * print out cpufreq information
 443  *
 444  * Write out information from cpufreq_driver->policy[cpu]; object must be
 445  * "unsigned int".
 446  */
 447
 448 #define show_one(file_name, object)                     \
 449 static ssize_t show_##file_name                         \
 450 (struct cpufreq_policy *policy, char *buf)              \
 451 {                                                       \
 452         return sprintf(buf, "%u\n", policy->object);    \
 453 }
 454
 455 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 456 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 457 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
 458 show_one(scaling_min_freq, min);
 459 show_one(scaling_max_freq, max);
 460 show_one(scaling_cur_freq, cur);
 461
 462 static int __cpufreq_set_policy(struct cpufreq_policy *data,
 463                                 struct cpufreq_policy *policy);
 464
 465 /**
 466  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 467  */
 468 #define store_one(file_name, object)                    \
 469 static ssize_t store_##file_name                                        \
 470 (struct cpufreq_policy *policy, const char *buf, size_t count)          \
 471 {                                                                       \
 472         unsigned int ret;                                               \
 473         struct cpufreq_policy new_policy;                               \
 474                                                                         \
 475         ret = cpufreq_get_policy(&new_policy, policy->cpu);             \
 476         if (ret)                                                        \
 477                 return -EINVAL;                                         \
 478                                                                         \
 479         ret = sscanf(buf, "%u", &new_policy.object);                    \
 480         if (ret != 1)                                                   \
 481                 return -EINVAL;                                         \
 482                                                                         \
 483         ret = __cpufreq_set_policy(policy, &new_policy);                \
 484         policy->user_policy.object = policy->object;                    \
 485                                                                         \
 486         return ret ? ret : count;                                       \
 487 }
 488
 489 store_one(scaling_min_freq, min);
 490 store_one(scaling_max_freq, max);
 491
 492 /**
 493  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 494  */
 495 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
 496                                         char *buf)
 497 {
 498         unsigned int cur_freq = __cpufreq_get(policy->cpu);
 499         if (!cur_freq)
 500                 return sprintf(buf, "<unknown>");
 501         return sprintf(buf, "%u\n", cur_freq);
 502 }
 503
 504 /**
 505  * show_scaling_governor - show the current policy for the specified CPU
 506  */
 507 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
 508 {
 509         if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
 510                 return sprintf(buf, "powersave\n");
 511         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 512                 return sprintf(buf, "performance\n");
 513         else if (policy->governor)
 514                 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
 515                                 policy->governor->name);
 516         return -EINVAL;
 517 }
 518
 519 /**
 520  * store_scaling_governor - store policy for the specified CPU
 521  */
 522 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
 523                                         const char *buf, size_t count)
 524 {
 525         unsigned int ret;
 526         char    str_governor[16];
 527         struct cpufreq_policy new_policy;
 528
 529         ret = cpufreq_get_policy(&new_policy, policy->cpu);
 530         if (ret)
 531                 return ret;
 532
 533         ret = sscanf(buf, "%15s", str_governor);
 534         if (ret != 1)
 535                 return -EINVAL;
 536
 537         if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 538                                                 &new_policy.governor))
 539                 return -EINVAL;
 540
 541         /*
 542          * Do not use cpufreq_set_policy here or the user_policy.max
 543          * will be wrongly overridden
 544          */
 545         ret = __cpufreq_set_policy(policy, &new_policy);
 546
 547         policy->user_policy.policy = policy->policy;
 548         policy->user_policy.governor = policy->governor;
 549
 550         if (ret)
 551                 return ret;
 552         else
 553                 return count;
 554 }
 555
 556 /**
 557  * show_scaling_driver - show the cpufreq driver currently loaded
 558  */
 559 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
 560 {
 561         return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
 562 }
 563
 564 /**
 565  * show_scaling_available_governors - show the available CPUfreq governors
 566  */
 567 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
 568                                                 char *buf)
 569 {
 570         ssize_t i = 0;
 571         struct cpufreq_governor *t;
 572
 573         if (!cpufreq_driver->target) {
 574                 i += sprintf(buf, "performance powersave");
 575                 goto out;
 576         }
 577
 578         list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
 579                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
 580                     - (CPUFREQ_NAME_LEN + 2)))
 581                         goto out;
 582                 i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
 583         }
 584 out:
 585         i += sprintf(&buf[i], "\n");
 586         return i;
 587 }
 588
 589 ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
 590 {
 591         ssize_t i = 0;
 592         unsigned int cpu;
 593
 594         for_each_cpu(cpu, mask) {
 595                 if (i)
 596                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 597                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 598                 if (i >= (PAGE_SIZE - 5))
 599                         break;
 600         }
 601         i += sprintf(&buf[i], "\n");
 602         return i;
 603 }
 604 EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
 605
 606 /**
 607  * show_related_cpus - show the CPUs affected by each transition even if
 608  * hw coordination is in use
 609  */
 610 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
 611 {
 612         return cpufreq_show_cpus(policy->related_cpus, buf);
 613 }
 614
 615 /**
 616  * show_affected_cpus - show the CPUs affected by each transition
 617  */
 618 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
 619 {
 620         return cpufreq_show_cpus(policy->cpus, buf);
 621 }
 622
 623 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 624                                         const char *buf, size_t count)
 625 {
 626         unsigned int freq = 0;
 627         unsigned int ret;
 628
 629         if (!policy->governor || !policy->governor->store_setspeed)
 630                 return -EINVAL;
 631
 632         ret = sscanf(buf, "%u", &freq);
 633         if (ret != 1)
 634                 return -EINVAL;
 635
 636         policy->governor->store_setspeed(policy, freq);
 637
 638         return count;
 639 }
 640
 641 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 642 {
 643         if (!policy->governor || !policy->governor->show_setspeed)
 644                 return sprintf(buf, "<unsupported>\n");
 645
 646         return policy->governor->show_setspeed(policy, buf);
 647 }
 648
 649 /**
 650  * show_bios_limit - show the current cpufreq HW/BIOS limitation
 651  */
 652 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
 653 {
 654         unsigned int limit;
 655         int ret;
 656         if (cpufreq_driver->bios_limit) {
 657                 ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
 658                 if (!ret)
 659                         return sprintf(buf, "%u\n", limit);
 660         }
 661         return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
 662 }
 663
 664 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
 665 cpufreq_freq_attr_ro(cpuinfo_min_freq);
 666 cpufreq_freq_attr_ro(cpuinfo_max_freq);
 667 cpufreq_freq_attr_ro(cpuinfo_transition_latency);
 668 cpufreq_freq_attr_ro(scaling_available_governors);
 669 cpufreq_freq_attr_ro(scaling_driver);
 670 cpufreq_freq_attr_ro(scaling_cur_freq);
 671 cpufreq_freq_attr_ro(bios_limit);
 672 cpufreq_freq_attr_ro(related_cpus);
 673 cpufreq_freq_attr_ro(affected_cpus);
 674 cpufreq_freq_attr_rw(scaling_min_freq);
 675 cpufreq_freq_attr_rw(scaling_max_freq);
 676 cpufreq_freq_attr_rw(scaling_governor);
 677 cpufreq_freq_attr_rw(scaling_setspeed);
 678
 679 static struct attribute *default_attrs[] = {
 680         &cpuinfo_min_freq.attr,
 681         &cpuinfo_max_freq.attr,
 682         &cpuinfo_transition_latency.attr,
 683         &scaling_min_freq.attr,
 684         &scaling_max_freq.attr,
 685         &affected_cpus.attr,
 686         &related_cpus.attr,
 687         &scaling_governor.attr,
 688         &scaling_driver.attr,
 689         &scaling_available_governors.attr,
 690         &scaling_setspeed.attr,
 691         NULL
 692 };
 693
 694 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
 695 #define to_attr(a) container_of(a, struct freq_attr, attr)
 696
 697 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
 698 {
 699         struct cpufreq_policy *policy = to_policy(kobj);
 700         struct freq_attr *fattr = to_attr(attr);
 701         ssize_t ret = -EINVAL;
 702         policy = cpufreq_cpu_get_sysfs(policy->cpu);
 703         if (!policy)
 704                 goto no_policy;
 705
 706         if (lock_policy_rwsem_read(policy->cpu) < 0)
 707                 goto fail;
 708
 709         if (fattr->show)
 710                 ret = fattr->show(policy, buf);
 711         else
 712                 ret = -EIO;
 713
 714         unlock_policy_rwsem_read(policy->cpu);
 715 fail:
 716         cpufreq_cpu_put_sysfs(policy);
 717 no_policy:
 718         return ret;
 719 }
 720
 721 static ssize_t store(struct kobject *kobj, struct attribute *attr,
 722                      const char *buf, size_t count)
 723 {
 724         struct cpufreq_policy *policy = to_policy(kobj);
 725         struct freq_attr *fattr = to_attr(attr);
 726         ssize_t ret = -EINVAL;
 727         policy = cpufreq_cpu_get_sysfs(policy->cpu);
 728         if (!policy)
 729                 goto no_policy;
 730
 731         if (lock_policy_rwsem_write(policy->cpu) < 0)
 732                 goto fail;
 733
 734         if (fattr->store)
 735                 ret = fattr->store(policy, buf, count);
 736         else
 737                 ret = -EIO;
 738
 739         unlock_policy_rwsem_write(policy->cpu);
 740 fail:
 741         cpufreq_cpu_put_sysfs(policy);
 742 no_policy:
 743         return ret;
 744 }
 745
 746 static void cpufreq_sysfs_release(struct kobject *kobj)
 747 {
 748         struct cpufreq_policy *policy = to_policy(kobj);
 749         pr_debug("last reference is dropped\n");
 750         complete(&policy->kobj_unregister);
 751 }
 752
 753 static const struct sysfs_ops sysfs_ops = {
 754         .show   = show,
 755         .store  = store,
 756 };
 757
 758 static struct kobj_type ktype_cpufreq = {
 759         .sysfs_ops      = &sysfs_ops,
 760         .default_attrs  = default_attrs,
 761         .release        = cpufreq_sysfs_release,
 762 };
 763
 764 struct kobject *cpufreq_global_kobject;
 765 EXPORT_SYMBOL(cpufreq_global_kobject);
 766
 767 static int cpufreq_global_kobject_usage;
 768
 769 int cpufreq_get_global_kobject(void)
 770 {
 771         if (!cpufreq_global_kobject_usage++)
 772                 return kobject_add(cpufreq_global_kobject,
 773                                 &cpu_subsys.dev_root->kobj, "%s", "cpufreq");
 774
 775         return 0;
 776 }
 777 EXPORT_SYMBOL(cpufreq_get_global_kobject);
 778
 779 void cpufreq_put_global_kobject(void)
 780 {
 781         if (!--cpufreq_global_kobject_usage)
 782                 kobject_del(cpufreq_global_kobject);
 783 }
 784 EXPORT_SYMBOL(cpufreq_put_global_kobject);
 785
 786 int cpufreq_sysfs_create_file(const struct attribute *attr)
 787 {
 788         int ret = cpufreq_get_global_kobject();
 789
 790         if (!ret) {
 791                 ret = sysfs_create_file(cpufreq_global_kobject, attr);
 792                 if (ret)
 793                         cpufreq_put_global_kobject();
 794         }
 795
 796         return ret;
 797 }
 798 EXPORT_SYMBOL(cpufreq_sysfs_create_file);
 799
 800 void cpufreq_sysfs_remove_file(const struct attribute *attr)
 801 {
 802         sysfs_remove_file(cpufreq_global_kobject, attr);
 803         cpufreq_put_global_kobject();
 804 }
 805 EXPORT_SYMBOL(cpufreq_sysfs_remove_file);
 806
 807 /* symlink affected CPUs */
 808 static int cpufreq_add_dev_symlink(unsigned int cpu,
 809                                    struct cpufreq_policy *policy)
 810 {
 811         unsigned int j;
 812         int ret = 0;
 813
 814         for_each_cpu(j, policy->cpus) {
 815                 struct cpufreq_policy *managed_policy;
 816                 struct device *cpu_dev;
 817
 818                 if (j == cpu)
 819                         continue;
 820
 821                 pr_debug("CPU %u already managed, adding link\n", j);
 822                 managed_policy = cpufreq_cpu_get(cpu);
 823                 cpu_dev = get_cpu_device(j);
 824                 ret = sysfs_create_link(&cpu_dev->kobj, &policy->kobj,
 825                                         "cpufreq");
 826                 if (ret) {
 827                         cpufreq_cpu_put(managed_policy);
 828                         return ret;
 829                 }
 830         }
 831         return ret;
 832 }
 833
 834 static int cpufreq_add_dev_interface(unsigned int cpu,
 835                                      struct cpufreq_policy *policy,
 836                                      struct device *dev)
 837 {
 838         struct freq_attr **drv_attr;
 839         int ret = 0;
 840
 841         /* prepare interface data */
 842         ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
 843                                    &dev->kobj, "cpufreq");
 844         if (ret)
 845                 return ret;
 846
 847         /* set up files for this cpu device */
 848         drv_attr = cpufreq_driver->attr;
 849         while ((drv_attr) && (*drv_attr)) {
 850                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
 851                 if (ret)
 852                         goto err_out_kobj_put;
 853                 drv_attr++;
 854         }
 855         if (cpufreq_driver->get) {
 856                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
 857                 if (ret)
 858                         goto err_out_kobj_put;
 859         }
 860         if (cpufreq_driver->target) {
 861                 ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
 862                 if (ret)
 863                         goto err_out_kobj_put;
 864         }
 865         if (cpufreq_driver->bios_limit) {
 866                 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
 867                 if (ret)
 868                         goto err_out_kobj_put;
 869         }
 870
 871         ret = cpufreq_add_dev_symlink(cpu, policy);
 872         if (ret)
 873                 goto err_out_kobj_put;
 874
 875         return ret;
 876
 877 err_out_kobj_put:
 878         kobject_put(&policy->kobj);
 879         wait_for_completion(&policy->kobj_unregister);
 880         return ret;
 881 }
 882
 883 static void cpufreq_init_policy(struct cpufreq_policy *policy)
 884 {
 885         struct cpufreq_policy new_policy;
 886         int ret = 0;
 887
 888         memcpy(&new_policy, policy, sizeof(struct cpufreq_policy));
 889         /* assure that the starting sequence is run in __cpufreq_set_policy */
 890         policy->governor = NULL;
 891
 892         /* set default policy */
 893         ret = __cpufreq_set_policy(policy, &new_policy);
 894         policy->user_policy.policy = policy->policy;
 895         policy->user_policy.governor = policy->governor;
 896
 897         if (ret) {
 898                 pr_debug("setting policy failed\n");
 899                 if (cpufreq_driver->exit)
 900                         cpufreq_driver->exit(policy);
 901         }
 902 }
 903
 904 #ifdef CONFIG_HOTPLUG_CPU
 905 static int cpufreq_add_policy_cpu(unsigned int cpu, unsigned int sibling,
 906                                   struct device *dev)
 907 {
 908         struct cpufreq_policy *policy;
 909         int ret = 0, has_target = !!cpufreq_driver->target;
 910         unsigned long flags;
 911
 912         policy = cpufreq_cpu_get(sibling);
 913         WARN_ON(!policy);
 914
 915         if (has_target)
 916                 __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
 917
 918         lock_policy_rwsem_write(sibling);
 919
 920         write_lock_irqsave(&cpufreq_driver_lock, flags);
 921
 922         cpumask_set_cpu(cpu, policy->cpus);
 923         per_cpu(cpufreq_policy_cpu, cpu) = policy->cpu;
 924         per_cpu(cpufreq_cpu_data, cpu) = policy;
 925         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
 926
 927         unlock_policy_rwsem_write(sibling);
 928
 929         if (has_target) {
 930                 __cpufreq_governor(policy, CPUFREQ_GOV_START);
 931                 __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
 932         }
 933
 934         ret = sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq");
 935         if (ret) {
 936                 cpufreq_cpu_put(policy);
 937                 return ret;
 938         }
 939
 940         return 0;
 941 }
 942 #endif
 943
 944 static struct cpufreq_policy *cpufreq_policy_alloc(void)
 945 {
 946         struct cpufreq_policy *policy;
 947
 948         policy = kzalloc(sizeof(*policy), GFP_KERNEL);
 949         if (!policy)
 950                 return NULL;
 951
 952         if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
 953                 goto err_free_policy;
 954
 955         if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
 956                 goto err_free_cpumask;
 957
 958         return policy;
 959
 960 err_free_cpumask:
 961         free_cpumask_var(policy->cpus);
 962 err_free_policy:
 963         kfree(policy);
 964
 965         return NULL;
 966 }
 967
 968 static void cpufreq_policy_free(struct cpufreq_policy *policy)
 969 {
 970         free_cpumask_var(policy->related_cpus);
 971         free_cpumask_var(policy->cpus);
 972         kfree(policy);
 973 }
 974
 975 /**
 976  * cpufreq_add_dev - add a CPU device
 977  *
 978  * Adds the cpufreq interface for a CPU device.
 979  *
 980  * The Oracle says: try running cpufreq registration/unregistration concurrently
 981  * with with cpu hotplugging and all hell will break loose. Tried to clean this
 982  * mess up, but more thorough testing is needed. - Mathieu
 983  */
 984 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
 985 {
 986         unsigned int j, cpu = dev->id;
 987         int ret = -ENOMEM;
 988         struct cpufreq_policy *policy;
 989         unsigned long flags;
 990 #ifdef CONFIG_HOTPLUG_CPU
 991         struct cpufreq_governor *gov;
 992         int sibling;
 993 #endif
 994
 995         if (cpu_is_offline(cpu))
 996                 return 0;
 997
 998         pr_debug("adding CPU %u\n", cpu);
 999
1000 #ifdef CONFIG_SMP
1001         /* check whether a different CPU already registered this
1002          * CPU because it is in the same boat. */
1003         policy = cpufreq_cpu_get(cpu);
1004         if (unlikely(policy)) {
1005                 cpufreq_cpu_put(policy);
1006                 return 0;
1007         }
1008
1009 #ifdef CONFIG_HOTPLUG_CPU
1010         /* Check if this cpu was hot-unplugged earlier and has siblings */
1011         read_lock_irqsave(&cpufreq_driver_lock, flags);
1012         for_each_online_cpu(sibling) {
1013                 struct cpufreq_policy *cp = per_cpu(cpufreq_cpu_data, sibling);
1014                 if (cp && cpumask_test_cpu(cpu, cp->related_cpus)) {
1015                         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1016                         return cpufreq_add_policy_cpu(cpu, sibling, dev);
1017                 }
1018         }
1019         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1020 #endif
1021 #endif
1022
1023         if (!try_module_get(cpufreq_driver->owner)) {
1024                 ret = -EINVAL;
1025                 goto module_out;
1026         }
1027
1028         policy = cpufreq_policy_alloc();
1029         if (!policy)
1030                 goto nomem_out;
1031
1032         policy->cpu = cpu;
1033         policy->governor = CPUFREQ_DEFAULT_GOVERNOR;
1034         cpumask_copy(policy->cpus, cpumask_of(cpu));
1035
1036         /* Initially set CPU itself as the policy_cpu */
1037         per_cpu(cpufreq_policy_cpu, cpu) = cpu;
1038
1039         init_completion(&policy->kobj_unregister);
1040         INIT_WORK(&policy->update, handle_update);
1041
1042         /* call driver. From then on the cpufreq must be able
1043          * to accept all calls to ->verify and ->setpolicy for this CPU
1044          */
1045         ret = cpufreq_driver->init(policy);
1046         if (ret) {
1047                 pr_debug("initialization failed\n");
1048                 goto err_set_policy_cpu;
1049         }
1050
1051         /* related cpus should atleast have policy->cpus */
1052         cpumask_or(policy->related_cpus, policy->related_cpus, policy->cpus);
1053
1054         /*
1055          * affected cpus must always be the one, which are online. We aren't
1056          * managing offline cpus here.
1057          */
1058         cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
1059
1060         policy->user_policy.min = policy->min;
1061         policy->user_policy.max = policy->max;
1062
1063         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1064                                      CPUFREQ_START, policy);
1065
1066 #ifdef CONFIG_HOTPLUG_CPU
1067         gov = __find_governor(per_cpu(cpufreq_cpu_governor, cpu));
1068         if (gov) {
1069                 policy->governor = gov;
1070                 pr_debug("Restoring governor %s for cpu %d\n",
1071                        policy->governor->name, cpu);
1072         }
1073 #endif
1074
1075         write_lock_irqsave(&cpufreq_driver_lock, flags);
1076         for_each_cpu(j, policy->cpus) {
1077                 per_cpu(cpufreq_cpu_data, j) = policy;
1078                 per_cpu(cpufreq_policy_cpu, j) = policy->cpu;
1079         }
1080         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1081
1082         ret = cpufreq_add_dev_interface(cpu, policy, dev);
1083         if (ret)
1084                 goto err_out_unregister;
1085
1086         cpufreq_init_policy(policy);
1087
1088         kobject_uevent(&policy->kobj, KOBJ_ADD);
1089         module_put(cpufreq_driver->owner);
1090         pr_debug("initialization complete\n");
1091
1092         return 0;
1093
1094 err_out_unregister:
1095         write_lock_irqsave(&cpufreq_driver_lock, flags);
1096         for_each_cpu(j, policy->cpus) {
1097                 per_cpu(cpufreq_cpu_data, j) = NULL;
1098                 if (j != cpu)
1099                         per_cpu(cpufreq_policy_cpu, j) = -1;
1100         }
1101         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1102
1103         kobject_put(&policy->kobj);
1104         wait_for_completion(&policy->kobj_unregister);
1105
1106 err_set_policy_cpu:
1107         per_cpu(cpufreq_policy_cpu, cpu) = -1;
1108         cpufreq_policy_free(policy);
1109 nomem_out:
1110         module_put(cpufreq_driver->owner);
1111 module_out:
1112         return ret;
1113 }
1114
1115 static void update_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu)
1116 {
1117         int j;
1118
1119         policy->last_cpu = policy->cpu;
1120         policy->cpu = cpu;
1121
1122         for_each_cpu(j, policy->cpus)
1123                 per_cpu(cpufreq_policy_cpu, j) = cpu;
1124
1125 #ifdef CONFIG_CPU_FREQ_TABLE
1126         cpufreq_frequency_table_update_policy_cpu(policy);
1127 #endif
1128         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1129                         CPUFREQ_UPDATE_POLICY_CPU, policy);
1130 }
1131
1132 /**
1133  * __cpufreq_remove_dev - remove a CPU device
1134  *
1135  * Removes the cpufreq interface for a CPU device.
1136  * Caller should already have policy_rwsem in write mode for this CPU.
1137  * This routine frees the rwsem before returning.
1138  */
1139 static int __cpufreq_remove_dev(struct device *dev,
1140                 struct subsys_interface *sif)
1141 {
1142         unsigned int cpu = dev->id, ret, cpus;
1143         unsigned long flags;
1144         struct cpufreq_policy *data;
1145         struct kobject *kobj;
1146         struct completion *cmp;
1147         struct device *cpu_dev;
1148
1149         pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
1150
1151         write_lock_irqsave(&cpufreq_driver_lock, flags);
1152
1153         data = per_cpu(cpufreq_cpu_data, cpu);
1154         per_cpu(cpufreq_cpu_data, cpu) = NULL;
1155
1156         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1157
1158         if (!data) {
1159                 pr_debug("%s: No cpu_data found\n", __func__);
1160                 return -EINVAL;
1161         }
1162
1163         if (cpufreq_driver->target)
1164                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1165
1166 #ifdef CONFIG_HOTPLUG_CPU
1167         if (!cpufreq_driver->setpolicy)
1168                 strncpy(per_cpu(cpufreq_cpu_governor, cpu),
1169                         data->governor->name, CPUFREQ_NAME_LEN);
1170 #endif
1171
1172         WARN_ON(lock_policy_rwsem_write(cpu));
1173         cpus = cpumask_weight(data->cpus);
1174
1175         if (cpus > 1)
1176                 cpumask_clear_cpu(cpu, data->cpus);
1177         unlock_policy_rwsem_write(cpu);
1178
1179         if (cpu != data->cpu) {
1180                 sysfs_remove_link(&dev->kobj, "cpufreq");
1181         } else if (cpus > 1) {
1182                 /* first sibling now owns the new sysfs dir */
1183                 cpu_dev = get_cpu_device(cpumask_first(data->cpus));
1184                 sysfs_remove_link(&cpu_dev->kobj, "cpufreq");
1185                 ret = kobject_move(&data->kobj, &cpu_dev->kobj);
1186                 if (ret) {
1187                         pr_err("%s: Failed to move kobj: %d", __func__, ret);
1188
1189                         WARN_ON(lock_policy_rwsem_write(cpu));
1190                         cpumask_set_cpu(cpu, data->cpus);
1191
1192                         write_lock_irqsave(&cpufreq_driver_lock, flags);
1193                         per_cpu(cpufreq_cpu_data, cpu) = data;
1194                         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1195
1196                         unlock_policy_rwsem_write(cpu);
1197
1198                         ret = sysfs_create_link(&cpu_dev->kobj, &data->kobj,
1199                                         "cpufreq");
1200                         return -EINVAL;
1201                 }
1202
1203                 WARN_ON(lock_policy_rwsem_write(cpu));
1204                 update_policy_cpu(data, cpu_dev->id);
1205                 unlock_policy_rwsem_write(cpu);
1206                 pr_debug("%s: policy Kobject moved to cpu: %d from: %d\n",
1207                                 __func__, cpu_dev->id, cpu);
1208         }
1209
1210         /* If cpu is last user of policy, free policy */
1211         if (cpus == 1) {
1212                 if (cpufreq_driver->target)
1213                         __cpufreq_governor(data, CPUFREQ_GOV_POLICY_EXIT);
1214
1215                 lock_policy_rwsem_read(cpu);
1216                 kobj = &data->kobj;
1217                 cmp = &data->kobj_unregister;
1218                 unlock_policy_rwsem_read(cpu);
1219                 kobject_put(kobj);
1220
1221                 /* we need to make sure that the underlying kobj is actually
1222                  * not referenced anymore by anybody before we proceed with
1223                  * unloading.
1224                  */
1225                 pr_debug("waiting for dropping of refcount\n");
1226                 wait_for_completion(cmp);
1227                 pr_debug("wait complete\n");
1228
1229                 if (cpufreq_driver->exit)
1230                         cpufreq_driver->exit(data);
1231
1232                 cpufreq_policy_free(data);
1233         } else {
1234                 pr_debug("%s: removing link, cpu: %d\n", __func__, cpu);
1235                 cpufreq_cpu_put(data);
1236                 if (cpufreq_driver->target) {
1237                         __cpufreq_governor(data, CPUFREQ_GOV_START);
1238                         __cpufreq_governor(data, CPUFREQ_GOV_LIMITS);
1239                 }
1240         }
1241
1242         per_cpu(cpufreq_policy_cpu, cpu) = -1;
1243         return 0;
1244 }
1245
1246 static int cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1247 {
1248         unsigned int cpu = dev->id;
1249         int retval;
1250
1251         if (cpu_is_offline(cpu))
1252                 return 0;
1253
1254         retval = __cpufreq_remove_dev(dev, sif);
1255         return retval;
1256 }
1257
1258 static void handle_update(struct work_struct *work)
1259 {
1260         struct cpufreq_policy *policy =
1261                 container_of(work, struct cpufreq_policy, update);
1262         unsigned int cpu = policy->cpu;
1263         pr_debug("handle_update for cpu %u called\n", cpu);
1264         cpufreq_update_policy(cpu);
1265 }
1266
1267 /**
1268  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're
1269  *      in deep trouble.
1270  *      @cpu: cpu number
1271  *      @old_freq: CPU frequency the kernel thinks the CPU runs at
1272  *      @new_freq: CPU frequency the CPU actually runs at
1273  *
1274  *      We adjust to current frequency first, and need to clean up later.
1275  *      So either call to cpufreq_update_policy() or schedule handle_update()).
1276  */
1277 static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,
1278                                 unsigned int new_freq)
1279 {
1280         struct cpufreq_policy *policy;
1281         struct cpufreq_freqs freqs;
1282         unsigned long flags;
1283
1284         pr_debug("Warning: CPU frequency out of sync: cpufreq and timing "
1285                "core thinks of %u, is %u kHz.\n", old_freq, new_freq);
1286
1287         freqs.old = old_freq;
1288         freqs.new = new_freq;
1289
1290         read_lock_irqsave(&cpufreq_driver_lock, flags);
1291         policy = per_cpu(cpufreq_cpu_data, cpu);
1292         read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1293
1294         cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE);
1295         cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE);
1296 }
1297
1298 /**
1299  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1300  * @cpu: CPU number
1301  *
1302  * This is the last known freq, without actually getting it from the driver.
1303  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1304  */
1305 unsigned int cpufreq_quick_get(unsigned int cpu)
1306 {
1307         struct cpufreq_policy *policy;
1308         unsigned int ret_freq = 0;
1309
1310         if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
1311                 return cpufreq_driver->get(cpu);
1312
1313         policy = cpufreq_cpu_get(cpu);
1314         if (policy) {
1315                 ret_freq = policy->cur;
1316                 cpufreq_cpu_put(policy);
1317         }
1318
1319         return ret_freq;
1320 }
1321 EXPORT_SYMBOL(cpufreq_quick_get);
1322
1323 /**
1324  * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1325  * @cpu: CPU number
1326  *
1327  * Just return the max possible frequency for a given CPU.
1328  */
1329 unsigned int cpufreq_quick_get_max(unsigned int cpu)
1330 {
1331         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1332         unsigned int ret_freq = 0;
1333
1334         if (policy) {
1335                 ret_freq = policy->max;
1336                 cpufreq_cpu_put(policy);
1337         }
1338
1339         return ret_freq;
1340 }
1341 EXPORT_SYMBOL(cpufreq_quick_get_max);
1342
1343 static unsigned int __cpufreq_get(unsigned int cpu)
1344 {
1345         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1346         unsigned int ret_freq = 0;
1347
1348         if (!cpufreq_driver->get)
1349                 return ret_freq;
1350
1351         ret_freq = cpufreq_driver->get(cpu);
1352
1353         if (ret_freq && policy->cur &&
1354                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1355                 /* verify no discrepancy between actual and
1356                                         saved value exists */
1357                 if (unlikely(ret_freq != policy->cur)) {
1358                         cpufreq_out_of_sync(cpu, policy->cur, ret_freq);
1359                         schedule_work(&policy->update);
1360                 }
1361         }
1362
1363         return ret_freq;
1364 }
1365
1366 /**
1367  * cpufreq_get - get the current CPU frequency (in kHz)
1368  * @cpu: CPU number
1369  *
1370  * Get the CPU current (static) CPU frequency
1371  */
1372 unsigned int cpufreq_get(unsigned int cpu)
1373 {
1374         unsigned int ret_freq = 0;
1375         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1376
1377         if (!policy)
1378                 goto out;
1379
1380         if (unlikely(lock_policy_rwsem_read(cpu)))
1381                 goto out_policy;
1382
1383         ret_freq = __cpufreq_get(cpu);
1384
1385         unlock_policy_rwsem_read(cpu);
1386
1387 out_policy:
1388         cpufreq_cpu_put(policy);
1389 out:
1390         return ret_freq;
1391 }
1392 EXPORT_SYMBOL(cpufreq_get);
1393
1394 static struct subsys_interface cpufreq_interface = {
1395         .name           = "cpufreq",
1396         .subsys         = &cpu_subsys,
1397         .add_dev        = cpufreq_add_dev,
1398         .remove_dev     = cpufreq_remove_dev,
1399 };
1400
1401 /**
1402  * cpufreq_bp_suspend - Prepare the boot CPU for system suspend.
1403  *
1404  * This function is only executed for the boot processor.  The other CPUs
1405  * have been put offline by means of CPU hotplug.
1406  */
1407 static int cpufreq_bp_suspend(void)
1408 {
1409         int ret = 0;
1410
1411         int cpu = smp_processor_id();
1412         struct cpufreq_policy *cpu_policy;
1413
1414         pr_debug("suspending cpu %u\n", cpu);
1415
1416         /* If there's no policy for the boot CPU, we have nothing to do. */
1417         cpu_policy = cpufreq_cpu_get(cpu);
1418         if (!cpu_policy)
1419                 return 0;
1420
1421         if (cpufreq_driver->suspend) {
1422                 ret = cpufreq_driver->suspend(cpu_policy);
1423                 if (ret)
1424                         printk(KERN_ERR "cpufreq: suspend failed in ->suspend "
1425                                         "step on CPU %u\n", cpu_policy->cpu);
1426         }
1427
1428         cpufreq_cpu_put(cpu_policy);
1429         return ret;
1430 }
1431
1432 /**
1433  * cpufreq_bp_resume - Restore proper frequency handling of the boot CPU.
1434  *
1435  *      1.) resume CPUfreq hardware support (cpufreq_driver->resume())
1436  *      2.) schedule call cpufreq_update_policy() ASAP as interrupts are
1437  *          restored. It will verify that the current freq is in sync with
1438  *          what we believe it to be. This is a bit later than when it
1439  *          should be, but nonethteless it's better than calling
1440  *          cpufreq_driver->get() here which might re-enable interrupts...
1441  *
1442  * This function is only executed for the boot CPU.  The other CPUs have not
1443  * been turned on yet.
1444  */
1445 static void cpufreq_bp_resume(void)
1446 {
1447         int ret = 0;
1448
1449         int cpu = smp_processor_id();
1450         struct cpufreq_policy *cpu_policy;
1451
1452         pr_debug("resuming cpu %u\n", cpu);
1453
1454         /* If there's no policy for the boot CPU, we have nothing to do. */
1455         cpu_policy = cpufreq_cpu_get(cpu);
1456         if (!cpu_policy)
1457                 return;
1458
1459         if (cpufreq_driver->resume) {
1460                 ret = cpufreq_driver->resume(cpu_policy);
1461                 if (ret) {
1462                         printk(KERN_ERR "cpufreq: resume failed in ->resume "
1463                                         "step on CPU %u\n", cpu_policy->cpu);
1464                         goto fail;
1465                 }
1466         }
1467
1468         schedule_work(&cpu_policy->update);
1469
1470 fail:
1471         cpufreq_cpu_put(cpu_policy);
1472 }
1473
1474 static struct syscore_ops cpufreq_syscore_ops = {
1475         .suspend        = cpufreq_bp_suspend,
1476         .resume         = cpufreq_bp_resume,
1477 };
1478
1479 /**
1480  *      cpufreq_get_current_driver - return current driver's name
1481  *
1482  *      Return the name string of the currently loaded cpufreq driver
1483  *      or NULL, if none.
1484  */
1485 const char *cpufreq_get_current_driver(void)
1486 {
1487         if (cpufreq_driver)
1488                 return cpufreq_driver->name;
1489
1490         return NULL;
1491 }
1492 EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
1493
1494 /*********************************************************************
1495  *                     NOTIFIER LISTS INTERFACE                      *
1496  *********************************************************************/
1497
1498 /**
1499  *      cpufreq_register_notifier - register a driver with cpufreq
1500  *      @nb: notifier function to register
1501  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1502  *
1503  *      Add a driver to one of two lists: either a list of drivers that
1504  *      are notified about clock rate changes (once before and once after
1505  *      the transition), or a list of drivers that are notified about
1506  *      changes in cpufreq policy.
1507  *
1508  *      This function may sleep, and has the same return conditions as
1509  *      blocking_notifier_chain_register.
1510  */
1511 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1512 {
1513         int ret;
1514
1515         if (cpufreq_disabled())
1516                 return -EINVAL;
1517
1518         WARN_ON(!init_cpufreq_transition_notifier_list_called);
1519
1520         switch (list) {
1521         case CPUFREQ_TRANSITION_NOTIFIER:
1522                 ret = srcu_notifier_chain_register(
1523                                 &cpufreq_transition_notifier_list, nb);
1524                 break;
1525         case CPUFREQ_POLICY_NOTIFIER:
1526                 ret = blocking_notifier_chain_register(
1527                                 &cpufreq_policy_notifier_list, nb);
1528                 break;
1529         default:
1530                 ret = -EINVAL;
1531         }
1532
1533         return ret;
1534 }
1535 EXPORT_SYMBOL(cpufreq_register_notifier);
1536
1537 /**
1538  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1539  *      @nb: notifier block to be unregistered
1540  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1541  *
1542  *      Remove a driver from the CPU frequency notifier list.
1543  *
1544  *      This function may sleep, and has the same return conditions as
1545  *      blocking_notifier_chain_unregister.
1546  */
1547 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1548 {
1549         int ret;
1550
1551         if (cpufreq_disabled())
1552                 return -EINVAL;
1553
1554         switch (list) {
1555         case CPUFREQ_TRANSITION_NOTIFIER:
1556                 ret = srcu_notifier_chain_unregister(
1557                                 &cpufreq_transition_notifier_list, nb);
1558                 break;
1559         case CPUFREQ_POLICY_NOTIFIER:
1560                 ret = blocking_notifier_chain_unregister(
1561                                 &cpufreq_policy_notifier_list, nb);
1562                 break;
1563         default:
1564                 ret = -EINVAL;
1565         }
1566
1567         return ret;
1568 }
1569 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1570
1571
1572 /*********************************************************************
1573  *                              GOVERNORS                            *
1574  *********************************************************************/
1575
1576 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1577                             unsigned int target_freq,
1578                             unsigned int relation)
1579 {
1580         int retval = -EINVAL;
1581         unsigned int old_target_freq = target_freq;
1582
1583         if (cpufreq_disabled())
1584                 return -ENODEV;
1585         if (policy->transition_ongoing)
1586                 return -EBUSY;
1587
1588         /* Make sure that target_freq is within supported range */
1589         if (target_freq > policy->max)
1590                 target_freq = policy->max;
1591         if (target_freq < policy->min)
1592                 target_freq = policy->min;
1593
1594         pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
1595                         policy->cpu, target_freq, relation, old_target_freq);
1596
1597         if (target_freq == policy->cur)
1598                 return 0;
1599
1600         if (cpufreq_driver->target)
1601                 retval = cpufreq_driver->target(policy, target_freq, relation);
1602
1603         return retval;
1604 }
1605 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1606
1607 int cpufreq_driver_target(struct cpufreq_policy *policy,
1608                           unsigned int target_freq,
1609                           unsigned int relation)
1610 {
1611         int ret = -EINVAL;
1612
1613         if (unlikely(lock_policy_rwsem_write(policy->cpu)))
1614                 goto fail;
1615
1616         ret = __cpufreq_driver_target(policy, target_freq, relation);
1617
1618         unlock_policy_rwsem_write(policy->cpu);
1619
1620 fail:
1621         return ret;
1622 }
1623 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
1624
1625 int __cpufreq_driver_getavg(struct cpufreq_policy *policy, unsigned int cpu)
1626 {
1627         if (cpufreq_disabled())
1628                 return 0;
1629
1630         if (!cpufreq_driver->getavg)
1631                 return 0;
1632
1633         return cpufreq_driver->getavg(policy, cpu);
1634 }
1635 EXPORT_SYMBOL_GPL(__cpufreq_driver_getavg);
1636
1637 /*
1638  * when "event" is CPUFREQ_GOV_LIMITS
1639  */
1640
1641 static int __cpufreq_governor(struct cpufreq_policy *policy,
1642                                         unsigned int event)
1643 {
1644         int ret;
1645
1646         /* Only must be defined when default governor is known to have latency
1647            restrictions, like e.g. conservative or ondemand.
1648            That this is the case is already ensured in Kconfig
1649         */
1650 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
1651         struct cpufreq_governor *gov = &cpufreq_gov_performance;
1652 #else
1653         struct cpufreq_governor *gov = NULL;
1654 #endif
1655
1656         if (policy->governor->max_transition_latency &&
1657             policy->cpuinfo.transition_latency >
1658             policy->governor->max_transition_latency) {
1659                 if (!gov)
1660                         return -EINVAL;
1661                 else {
1662                         printk(KERN_WARNING "%s governor failed, too long"
1663                                " transition latency of HW, fallback"
1664                                " to %s governor\n",
1665                                policy->governor->name,
1666                                gov->name);
1667                         policy->governor = gov;
1668                 }
1669         }
1670
1671         if (!try_module_get(policy->governor->owner))
1672                 return -EINVAL;
1673
1674         pr_debug("__cpufreq_governor for CPU %u, event %u\n",
1675                                                 policy->cpu, event);
1676
1677         mutex_lock(&cpufreq_governor_lock);
1678         if ((!policy->governor_enabled && (event == CPUFREQ_GOV_STOP)) ||
1679             (policy->governor_enabled && (event == CPUFREQ_GOV_START))) {
1680                 mutex_unlock(&cpufreq_governor_lock);
1681                 return -EBUSY;
1682         }
1683
1684         if (event == CPUFREQ_GOV_STOP)
1685                 policy->governor_enabled = false;
1686         else if (event == CPUFREQ_GOV_START)
1687                 policy->governor_enabled = true;
1688
1689         mutex_unlock(&cpufreq_governor_lock);
1690
1691         ret = policy->governor->governor(policy, event);
1692
1693         if (!ret) {
1694                 if (event == CPUFREQ_GOV_POLICY_INIT)
1695                         policy->governor->initialized++;
1696                 else if (event == CPUFREQ_GOV_POLICY_EXIT)
1697                         policy->governor->initialized--;
1698         } else {
1699                 /* Restore original values */
1700                 mutex_lock(&cpufreq_governor_lock);
1701                 if (event == CPUFREQ_GOV_STOP)
1702                         policy->governor_enabled = true;
1703                 else if (event == CPUFREQ_GOV_START)
1704                         policy->governor_enabled = false;
1705                 mutex_unlock(&cpufreq_governor_lock);
1706         }
1707
1708         /* we keep one module reference alive for
1709                         each CPU governed by this CPU */
1710         if ((event != CPUFREQ_GOV_START) || ret)
1711                 module_put(policy->governor->owner);
1712         if ((event == CPUFREQ_GOV_STOP) && !ret)
1713                 module_put(policy->governor->owner);
1714
1715         return ret;
1716 }
1717
1718 int cpufreq_register_governor(struct cpufreq_governor *governor)
1719 {
1720         int err;
1721
1722         if (!governor)
1723                 return -EINVAL;
1724
1725         if (cpufreq_disabled())
1726                 return -ENODEV;
1727
1728         mutex_lock(&cpufreq_governor_mutex);
1729
1730         governor->initialized = 0;
1731         err = -EBUSY;
1732         if (__find_governor(governor->name) == NULL) {
1733                 err = 0;
1734                 list_add(&governor->governor_list, &cpufreq_governor_list);
1735         }
1736
1737         mutex_unlock(&cpufreq_governor_mutex);
1738         return err;
1739 }
1740 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
1741
1742 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
1743 {
1744 #ifdef CONFIG_HOTPLUG_CPU
1745         int cpu;
1746 #endif
1747
1748         if (!governor)
1749                 return;
1750
1751         if (cpufreq_disabled())
1752                 return;
1753
1754 #ifdef CONFIG_HOTPLUG_CPU
1755         for_each_present_cpu(cpu) {
1756                 if (cpu_online(cpu))
1757                         continue;
1758                 if (!strcmp(per_cpu(cpufreq_cpu_governor, cpu), governor->name))
1759                         strcpy(per_cpu(cpufreq_cpu_governor, cpu), "\0");
1760         }
1761 #endif
1762
1763         mutex_lock(&cpufreq_governor_mutex);
1764         list_del(&governor->governor_list);
1765         mutex_unlock(&cpufreq_governor_mutex);
1766         return;
1767 }
1768 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
1769
1770
1771 /*********************************************************************
1772  *                          POLICY INTERFACE                         *
1773  *********************************************************************/
1774
1775 /**
1776  * cpufreq_get_policy - get the current cpufreq_policy
1777  * @policy: struct cpufreq_policy into which the current cpufreq_policy
1778  *      is written
1779  *
1780  * Reads the current cpufreq policy.
1781  */
1782 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
1783 {
1784         struct cpufreq_policy *cpu_policy;
1785         if (!policy)
1786                 return -EINVAL;
1787
1788         cpu_policy = cpufreq_cpu_get(cpu);
1789         if (!cpu_policy)
1790                 return -EINVAL;
1791
1792         memcpy(policy, cpu_policy, sizeof(struct cpufreq_policy));
1793
1794         cpufreq_cpu_put(cpu_policy);
1795         return 0;
1796 }
1797 EXPORT_SYMBOL(cpufreq_get_policy);
1798
1799 /*
1800  * data   : current policy.
1801  * policy : policy to be set.
1802  */
1803 static int __cpufreq_set_policy(struct cpufreq_policy *data,
1804                                 struct cpufreq_policy *policy)
1805 {
1806         int ret = 0, failed = 1;
1807
1808         pr_debug("setting new policy for CPU %u: %u - %u kHz\n", policy->cpu,
1809                 policy->min, policy->max);
1810
1811         memcpy(&policy->cpuinfo, &data->cpuinfo,
1812                                 sizeof(struct cpufreq_cpuinfo));
1813
1814         if (policy->min > data->max || policy->max < data->min) {
1815                 ret = -EINVAL;
1816                 goto error_out;
1817         }
1818
1819         /* verify the cpu speed can be set within this limit */
1820         ret = cpufreq_driver->verify(policy);
1821         if (ret)
1822                 goto error_out;
1823
1824         /* adjust if necessary - all reasons */
1825         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1826                         CPUFREQ_ADJUST, policy);
1827
1828         /* adjust if necessary - hardware incompatibility*/
1829         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1830                         CPUFREQ_INCOMPATIBLE, policy);
1831
1832         /*
1833          * verify the cpu speed can be set within this limit, which might be
1834          * different to the first one
1835          */
1836         ret = cpufreq_driver->verify(policy);
1837         if (ret)
1838                 goto error_out;
1839
1840         /* notification of the new policy */
1841         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1842                         CPUFREQ_NOTIFY, policy);
1843
1844         data->min = policy->min;
1845         data->max = policy->max;
1846
1847         pr_debug("new min and max freqs are %u - %u kHz\n",
1848                                         data->min, data->max);
1849
1850         if (cpufreq_driver->setpolicy) {
1851                 data->policy = policy->policy;
1852                 pr_debug("setting range\n");
1853                 ret = cpufreq_driver->setpolicy(policy);
1854         } else {
1855                 if (policy->governor != data->governor) {
1856                         /* save old, working values */
1857                         struct cpufreq_governor *old_gov = data->governor;
1858
1859                         pr_debug("governor switch\n");
1860
1861                         /* end old governor */
1862                         if (data->governor) {
1863                                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1864                                 unlock_policy_rwsem_write(policy->cpu);
1865                                 __cpufreq_governor(data,
1866                                                 CPUFREQ_GOV_POLICY_EXIT);
1867                                 lock_policy_rwsem_write(policy->cpu);
1868                         }
1869
1870                         /* start new governor */
1871                         data->governor = policy->governor;
1872                         if (!__cpufreq_governor(data, CPUFREQ_GOV_POLICY_INIT)) {
1873                                 if (!__cpufreq_governor(data, CPUFREQ_GOV_START)) {
1874                                         failed = 0;
1875                                 } else {
1876                                         unlock_policy_rwsem_write(policy->cpu);
1877                                         __cpufreq_governor(data,
1878                                                         CPUFREQ_GOV_POLICY_EXIT);
1879                                         lock_policy_rwsem_write(policy->cpu);
1880                                 }
1881                         }
1882
1883                         if (failed) {
1884                                 /* new governor failed, so re-start old one */
1885                                 pr_debug("starting governor %s failed\n",
1886                                                         data->governor->name);
1887                                 if (old_gov) {
1888                                         data->governor = old_gov;
1889                                         __cpufreq_governor(data,
1890                                                         CPUFREQ_GOV_POLICY_INIT);
1891                                         __cpufreq_governor(data,
1892                                                            CPUFREQ_GOV_START);
1893                                 }
1894                                 ret = -EINVAL;
1895                                 goto error_out;
1896                         }
1897                         /* might be a policy change, too, so fall through */
1898                 }
1899                 pr_debug("governor: change or update limits\n");
1900                 __cpufreq_governor(data, CPUFREQ_GOV_LIMITS);
1901         }
1902
1903 error_out:
1904         return ret;
1905 }
1906
1907 /**
1908  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
1909  *      @cpu: CPU which shall be re-evaluated
1910  *
1911  *      Useful for policy notifiers which have different necessities
1912  *      at different times.
1913  */
1914 int cpufreq_update_policy(unsigned int cpu)
1915 {
1916         struct cpufreq_policy *data = cpufreq_cpu_get(cpu);
1917         struct cpufreq_policy policy;
1918         int ret;
1919
1920         if (!data) {
1921                 ret = -ENODEV;
1922                 goto no_policy;
1923         }
1924
1925         if (unlikely(lock_policy_rwsem_write(cpu))) {
1926                 ret = -EINVAL;
1927                 goto fail;
1928         }
1929
1930         pr_debug("updating policy for CPU %u\n", cpu);
1931         memcpy(&policy, data, sizeof(struct cpufreq_policy));
1932         policy.min = data->user_policy.min;
1933         policy.max = data->user_policy.max;
1934         policy.policy = data->user_policy.policy;
1935         policy.governor = data->user_policy.governor;
1936
1937         /*
1938          * BIOS might change freq behind our back
1939          * -> ask driver for current freq and notify governors about a change
1940          */
1941         if (cpufreq_driver->get) {
1942                 policy.cur = cpufreq_driver->get(cpu);
1943                 if (!data->cur) {
1944                         pr_debug("Driver did not initialize current freq");
1945                         data->cur = policy.cur;
1946                 } else {
1947                         if (data->cur != policy.cur && cpufreq_driver->target)
1948                                 cpufreq_out_of_sync(cpu, data->cur,
1949                                                                 policy.cur);
1950                 }
1951         }
1952
1953         ret = __cpufreq_set_policy(data, &policy);
1954
1955         unlock_policy_rwsem_write(cpu);
1956
1957 fail:
1958         cpufreq_cpu_put(data);
1959 no_policy:
1960         return ret;
1961 }
1962 EXPORT_SYMBOL(cpufreq_update_policy);
1963
1964 static int cpufreq_cpu_callback(struct notifier_block *nfb,
1965                                         unsigned long action, void *hcpu)
1966 {
1967         unsigned int cpu = (unsigned long)hcpu;
1968         struct device *dev;
1969
1970         dev = get_cpu_device(cpu);
1971         if (dev) {
1972                 switch (action) {
1973                 case CPU_ONLINE:
1974                 case CPU_ONLINE_FROZEN:
1975                         cpufreq_add_dev(dev, NULL);
1976                         cpufreq_update_policy(cpu);
1977                         break;
1978                 case CPU_DOWN_PREPARE:
1979                 case CPU_DOWN_PREPARE_FROZEN:
1980                         __cpufreq_remove_dev(dev, NULL);
1981                         break;
1982                 case CPU_DOWN_FAILED:
1983                 case CPU_DOWN_FAILED_FROZEN:
1984                         cpufreq_add_dev(dev, NULL);
1985                         break;
1986                 }
1987         }
1988         return NOTIFY_OK;
1989 }
1990
1991 static struct notifier_block __refdata cpufreq_cpu_notifier = {
1992         .notifier_call = cpufreq_cpu_callback,
1993 };
1994
1995 /*********************************************************************
1996  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
1997  *********************************************************************/
1998
1999 /**
2000  * cpufreq_register_driver - register a CPU Frequency driver
2001  * @driver_data: A struct cpufreq_driver containing the values#
2002  * submitted by the CPU Frequency driver.
2003  *
2004  * Registers a CPU Frequency driver to this core code. This code
2005  * returns zero on success, -EBUSY when another driver got here first
2006  * (and isn't unregistered in the meantime).
2007  *
2008  */
2009 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
2010 {
2011         unsigned long flags;
2012         int ret;
2013
2014         if (cpufreq_disabled())
2015                 return -ENODEV;
2016
2017         if (!driver_data || !driver_data->verify || !driver_data->init ||
2018             ((!driver_data->setpolicy) && (!driver_data->target)))
2019                 return -EINVAL;
2020
2021         pr_debug("trying to register driver %s\n", driver_data->name);
2022
2023         if (driver_data->setpolicy)
2024                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
2025
2026         write_lock_irqsave(&cpufreq_driver_lock, flags);
2027         if (cpufreq_driver) {
2028                 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2029                 return -EBUSY;
2030         }
2031         cpufreq_driver = driver_data;
2032         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2033
2034         ret = subsys_interface_register(&cpufreq_interface);
2035         if (ret)
2036                 goto err_null_driver;
2037
2038         if (!(cpufreq_driver->flags & CPUFREQ_STICKY)) {
2039                 int i;
2040                 ret = -ENODEV;
2041
2042                 /* check for at least one working CPU */
2043                 for (i = 0; i < nr_cpu_ids; i++)
2044                         if (cpu_possible(i) && per_cpu(cpufreq_cpu_data, i)) {
2045                                 ret = 0;
2046                                 break;
2047                         }
2048
2049                 /* if all ->init() calls failed, unregister */
2050                 if (ret) {
2051                         pr_debug("no CPU initialized for driver %s\n",
2052                                                         driver_data->name);
2053                         goto err_if_unreg;
2054                 }
2055         }
2056
2057         register_hotcpu_notifier(&cpufreq_cpu_notifier);
2058         pr_debug("driver %s up and running\n", driver_data->name);
2059
2060         return 0;
2061 err_if_unreg:
2062         subsys_interface_unregister(&cpufreq_interface);
2063 err_null_driver:
2064         write_lock_irqsave(&cpufreq_driver_lock, flags);
2065         cpufreq_driver = NULL;
2066         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2067         return ret;
2068 }
2069 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
2070
2071 /**
2072  * cpufreq_unregister_driver - unregister the current CPUFreq driver
2073  *
2074  * Unregister the current CPUFreq driver. Only call this if you have
2075  * the right to do so, i.e. if you have succeeded in initialising before!
2076  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
2077  * currently not initialised.
2078  */
2079 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
2080 {
2081         unsigned long flags;
2082
2083         if (!cpufreq_driver || (driver != cpufreq_driver))
2084                 return -EINVAL;
2085
2086         pr_debug("unregistering driver %s\n", driver->name);
2087
2088         subsys_interface_unregister(&cpufreq_interface);
2089         unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
2090
2091         write_lock_irqsave(&cpufreq_driver_lock, flags);
2092         cpufreq_driver = NULL;
2093         write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2094
2095         return 0;
2096 }
2097 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
2098
2099 static int __init cpufreq_core_init(void)
2100 {
2101         int cpu;
2102
2103         if (cpufreq_disabled())
2104                 return -ENODEV;
2105
2106         for_each_possible_cpu(cpu) {
2107                 per_cpu(cpufreq_policy_cpu, cpu) = -1;
2108                 init_rwsem(&per_cpu(cpu_policy_rwsem, cpu));
2109         }
2110
2111         cpufreq_global_kobject = kobject_create();
2112         BUG_ON(!cpufreq_global_kobject);
2113         register_syscore_ops(&cpufreq_syscore_ops);
2114
2115         return 0;
2116 }
2117 core_initcall(cpufreq_core_init);