drivers/cpufreq/cpufreq_interactive.c

   1 /*
   2  * drivers/cpufreq/cpufreq_interactive.c
   3  *
   4  * Copyright (C) 2010 Google, Inc.
   5  *
   6  * This software is licensed under the terms of the GNU General Public
   7  * License version 2, as published by the Free Software Foundation, and
   8  * may be copied, distributed, and modified under those terms.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * Author: Mike Chan (mike@android.com)
  16  *
  17  */
  18
  19 #include <linux/cpu.h>
  20 #include <linux/cpumask.h>
  21 #include <linux/cpufreq.h>
  22 #include <linux/module.h>
  23 #include <linux/mutex.h>
  24 #include <linux/sched.h>
  25 #include <linux/sched/rt.h>
  26 #include <linux/tick.h>
  27 #include <linux/time.h>
  28 #include <linux/timer.h>
  29 #include <linux/workqueue.h>
  30 #include <linux/kthread.h>
  31 #include <linux/mutex.h>
  32
  33 #define CREATE_TRACE_POINTS
  34 #include <trace/events/cpufreq_interactive.h>
  35
  36 #include <asm/cputime.h>
  37
  38 static atomic_t active_count = ATOMIC_INIT(0);
  39
  40 struct cpufreq_interactive_cpuinfo {
  41         struct timer_list cpu_timer;
  42         int timer_idlecancel;
  43         u64 time_in_idle;
  44         u64 idle_exit_time;
  45         u64 timer_run_time;
  46         int idling;
  47         u64 freq_change_time;
  48         u64 freq_change_time_in_idle;
  49         struct cpufreq_policy *policy;
  50         struct cpufreq_frequency_table *freq_table;
  51         unsigned int target_freq;
  52         int governor_enabled;
  53 };
  54
  55 static DEFINE_PER_CPU(struct cpufreq_interactive_cpuinfo, cpuinfo);
  56
  57 /* Workqueues handle frequency scaling */
  58 static struct task_struct *up_task;
  59 static struct workqueue_struct *down_wq;
  60 static struct work_struct freq_scale_down_work;
  61 static cpumask_t up_cpumask;
  62 static spinlock_t up_cpumask_lock;
  63 static cpumask_t down_cpumask;
  64 static spinlock_t down_cpumask_lock;
  65 static struct mutex set_speed_lock;
  66
  67 /* Hi speed to bump to from lo speed when load burst (default max) */
  68 static u64 hispeed_freq;
  69
  70 /* Go to hi speed when CPU load at or above this value. */
  71 #define DEFAULT_GO_HISPEED_LOAD 95
  72 static unsigned long go_hispeed_load;
  73
  74 /*
  75  * The minimum amount of time to spend at a frequency before we can ramp down.
  76  */
  77 #define DEFAULT_MIN_SAMPLE_TIME 20 * USEC_PER_MSEC
  78 static unsigned long min_sample_time;
  79
  80 /*
  81  * The sample rate of the timer used to increase frequency
  82  */
  83 #define DEFAULT_TIMER_RATE 20 * USEC_PER_MSEC
  84 static unsigned long timer_rate;
  85
  86 static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
  87                 unsigned int event);
  88
  89 #ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE
  90 static
  91 #endif
  92 struct cpufreq_governor cpufreq_gov_interactive = {
  93         .name = "interactive",
  94         .governor = cpufreq_governor_interactive,
  95         .max_transition_latency = 10000000,
  96         .owner = THIS_MODULE,
  97 };
  98
  99 static void cpufreq_interactive_timer(unsigned long data)
 100 {
 101         unsigned int delta_idle;
 102         unsigned int delta_time;
 103         int cpu_load;
 104         int load_since_change;
 105         u64 time_in_idle;
 106         u64 idle_exit_time;
 107         struct cpufreq_interactive_cpuinfo *pcpu =
 108                 &per_cpu(cpuinfo, data);
 109         u64 now_idle;
 110         unsigned int new_freq;
 111         unsigned int index;
 112         unsigned long flags;
 113
 114         smp_rmb();
 115
 116         if (!pcpu->governor_enabled)
 117                 goto exit;
 118
 119         /*
 120          * Once pcpu->timer_run_time is updated to >= pcpu->idle_exit_time,
 121          * this lets idle exit know the current idle time sample has
 122          * been processed, and idle exit can generate a new sample and
 123          * re-arm the timer.  This prevents a concurrent idle
 124          * exit on that CPU from writing a new set of info at the same time
 125          * the timer function runs (the timer function can't use that info
 126          * until more time passes).
 127          */
 128         time_in_idle = pcpu->time_in_idle;
 129         idle_exit_time = pcpu->idle_exit_time;
 130         now_idle = get_cpu_idle_time_us(data, &pcpu->timer_run_time);
 131         smp_wmb();
 132
 133         /* If we raced with cancelling a timer, skip. */
 134         if (!idle_exit_time)
 135                 goto exit;
 136
 137         delta_idle = (unsigned int)(now_idle - time_in_idle);
 138         delta_time = (unsigned int)(pcpu->timer_run_time - idle_exit_time);
 139
 140         /*
 141          * If timer ran less than 1ms after short-term sample started, retry.
 142          */
 143         if (delta_time < 1000)
 144                 goto rearm;
 145
 146         if (delta_idle > delta_time)
 147                 cpu_load = 0;
 148         else
 149                 cpu_load = 100 * (delta_time - delta_idle) / delta_time;
 150
 151         delta_idle = (unsigned int)(now_idle - pcpu->freq_change_time_in_idle);
 152         delta_time = (unsigned int)(pcpu->timer_run_time - pcpu->freq_change_time);
 153
 154         if ((delta_time == 0) || (delta_idle > delta_time))
 155                 load_since_change = 0;
 156         else
 157                 load_since_change =
 158                         100 * (delta_time - delta_idle) / delta_time;
 159
 160         /*
 161          * Choose greater of short-term load (since last idle timer
 162          * started or timer function re-armed itself) or long-term load
 163          * (since last frequency change).
 164          */
 165         if (load_since_change > cpu_load)
 166                 cpu_load = load_since_change;
 167
 168         if (cpu_load >= go_hispeed_load) {
 169                 if (pcpu->policy->cur == pcpu->policy->min) {
 170                         new_freq = hispeed_freq;
 171                 } else {
 172                         new_freq = pcpu->policy->max * cpu_load / 100;
 173
 174                         if (new_freq < hispeed_freq)
 175                                 new_freq = hispeed_freq;
 176                 }
 177         } else {
 178                 new_freq = pcpu->policy->max * cpu_load / 100;
 179         }
 180
 181         if (cpufreq_frequency_table_target(pcpu->policy, pcpu->freq_table,
 182                                            new_freq, CPUFREQ_RELATION_H,
 183                                            &index)) {
 184                 pr_warn_once("timer %d: cpufreq_frequency_table_target error\n",
 185                              (int) data);
 186                 goto rearm;
 187         }
 188
 189         new_freq = pcpu->freq_table[index].frequency;
 190
 191         if (pcpu->target_freq == new_freq)
 192         {
 193                 trace_cpufreq_interactive_already(data, cpu_load,
 194                                                   pcpu->target_freq, new_freq);
 195                 goto rearm_if_notmax;
 196         }
 197
 198         /*
 199          * Do not scale down unless we have been at this frequency for the
 200          * minimum sample time.
 201          */
 202         if (new_freq < pcpu->target_freq) {
 203                 if (pcpu->timer_run_time - pcpu->freq_change_time
 204                     < min_sample_time) {
 205                         trace_cpufreq_interactive_notyet(data, cpu_load,
 206                                          pcpu->target_freq, new_freq);
 207                         goto rearm;
 208                 }
 209         }
 210
 211         trace_cpufreq_interactive_target(data, cpu_load, pcpu->target_freq,
 212                                          new_freq);
 213
 214         if (new_freq < pcpu->target_freq) {
 215                 pcpu->target_freq = new_freq;
 216                 spin_lock_irqsave(&down_cpumask_lock, flags);
 217                 cpumask_set_cpu(data, &down_cpumask);
 218                 spin_unlock_irqrestore(&down_cpumask_lock, flags);
 219                 queue_work(down_wq, &freq_scale_down_work);
 220         } else {
 221                 pcpu->target_freq = new_freq;
 222                 spin_lock_irqsave(&up_cpumask_lock, flags);
 223                 cpumask_set_cpu(data, &up_cpumask);
 224                 spin_unlock_irqrestore(&up_cpumask_lock, flags);
 225                 wake_up_process(up_task);
 226         }
 227
 228 rearm_if_notmax:
 229         /*
 230          * Already set max speed and don't see a need to change that,
 231          * wait until next idle to re-evaluate, don't need timer.
 232          */
 233         if (pcpu->target_freq == pcpu->policy->max)
 234                 goto exit;
 235
 236 rearm:
 237         if (!timer_pending(&pcpu->cpu_timer)) {
 238                 /*
 239                  * If already at min: if that CPU is idle, don't set timer.
 240                  * Else cancel the timer if that CPU goes idle.  We don't
 241                  * need to re-evaluate speed until the next idle exit.
 242                  */
 243                 if (pcpu->target_freq == pcpu->policy->min) {
 244                         smp_rmb();
 245
 246                         if (pcpu->idling)
 247                                 goto exit;
 248
 249                         pcpu->timer_idlecancel = 1;
 250                 }
 251
 252                 pcpu->time_in_idle = get_cpu_idle_time_us(
 253                         data, &pcpu->idle_exit_time);
 254                 mod_timer(&pcpu->cpu_timer,
 255                           jiffies + usecs_to_jiffies(timer_rate));
 256         }
 257
 258 exit:
 259         return;
 260 }
 261
 262 static void cpufreq_interactive_idle_start(void)
 263 {
 264         struct cpufreq_interactive_cpuinfo *pcpu =
 265                 &per_cpu(cpuinfo, smp_processor_id());
 266         int pending;
 267
 268         if (!pcpu->governor_enabled)
 269                 return;
 270
 271         pcpu->idling = 1;
 272         smp_wmb();
 273         pending = timer_pending(&pcpu->cpu_timer);
 274
 275         if (pcpu->target_freq != pcpu->policy->min) {
 276 #ifdef CONFIG_SMP
 277                 /*
 278                  * Entering idle while not at lowest speed.  On some
 279                  * platforms this can hold the other CPU(s) at that speed
 280                  * even though the CPU is idle. Set a timer to re-evaluate
 281                  * speed so this idle CPU doesn't hold the other CPUs above
 282                  * min indefinitely.  This should probably be a quirk of
 283                  * the CPUFreq driver.
 284                  */
 285                 if (!pending) {
 286                         pcpu->time_in_idle = get_cpu_idle_time_us(
 287                                 smp_processor_id(), &pcpu->idle_exit_time);
 288                         pcpu->timer_idlecancel = 0;
 289                         mod_timer(&pcpu->cpu_timer,
 290                                   jiffies + usecs_to_jiffies(timer_rate));
 291                 }
 292 #endif
 293         } else {
 294                 /*
 295                  * If at min speed and entering idle after load has
 296                  * already been evaluated, and a timer has been set just in
 297                  * case the CPU suddenly goes busy, cancel that timer.  The
 298                  * CPU didn't go busy; we'll recheck things upon idle exit.
 299                  */
 300                 if (pending && pcpu->timer_idlecancel) {
 301                         del_timer(&pcpu->cpu_timer);
 302                         /*
 303                          * Ensure last timer run time is after current idle
 304                          * sample start time, so next idle exit will always
 305                          * start a new idle sampling period.
 306                          */
 307                         pcpu->idle_exit_time = 0;
 308                         pcpu->timer_idlecancel = 0;
 309                 }
 310         }
 311
 312 }
 313
 314 static void cpufreq_interactive_idle_end(void)
 315 {
 316         struct cpufreq_interactive_cpuinfo *pcpu =
 317                 &per_cpu(cpuinfo, smp_processor_id());
 318
 319         pcpu->idling = 0;
 320         smp_wmb();
 321
 322         /*
 323          * Arm the timer for 1-2 ticks later if not already, and if the timer
 324          * function has already processed the previous load sampling
 325          * interval.  (If the timer is not pending but has not processed
 326          * the previous interval, it is probably racing with us on another
 327          * CPU.  Let it compute load based on the previous sample and then
 328          * re-arm the timer for another interval when it's done, rather
 329          * than updating the interval start time to be "now", which doesn't
 330          * give the timer function enough time to make a decision on this
 331          * run.)
 332          */
 333         if (timer_pending(&pcpu->cpu_timer) == 0 &&
 334             pcpu->timer_run_time >= pcpu->idle_exit_time &&
 335             pcpu->governor_enabled) {
 336                 pcpu->time_in_idle =
 337                         get_cpu_idle_time_us(smp_processor_id(),
 338                                              &pcpu->idle_exit_time);
 339                 pcpu->timer_idlecancel = 0;
 340                 mod_timer(&pcpu->cpu_timer,
 341                           jiffies + usecs_to_jiffies(timer_rate));
 342         }
 343
 344 }
 345
 346 static int cpufreq_interactive_up_task(void *data)
 347 {
 348         unsigned int cpu;
 349         cpumask_t tmp_mask;
 350         unsigned long flags;
 351         struct cpufreq_interactive_cpuinfo *pcpu;
 352
 353         while (1) {
 354                 set_current_state(TASK_INTERRUPTIBLE);
 355                 spin_lock_irqsave(&up_cpumask_lock, flags);
 356
 357                 if (cpumask_empty(&up_cpumask)) {
 358                         spin_unlock_irqrestore(&up_cpumask_lock, flags);
 359                         schedule();
 360
 361                         if (kthread_should_stop())
 362                                 break;
 363
 364                         spin_lock_irqsave(&up_cpumask_lock, flags);
 365                 }
 366
 367                 set_current_state(TASK_RUNNING);
 368                 tmp_mask = up_cpumask;
 369                 cpumask_clear(&up_cpumask);
 370                 spin_unlock_irqrestore(&up_cpumask_lock, flags);
 371
 372                 for_each_cpu(cpu, &tmp_mask) {
 373                         unsigned int j;
 374                         unsigned int max_freq = 0;
 375
 376                         pcpu = &per_cpu(cpuinfo, cpu);
 377                         smp_rmb();
 378
 379                         if (!pcpu->governor_enabled)
 380                                 continue;
 381
 382                         mutex_lock(&set_speed_lock);
 383
 384                         for_each_cpu(j, pcpu->policy->cpus) {
 385                                 struct cpufreq_interactive_cpuinfo *pjcpu =
 386                                         &per_cpu(cpuinfo, j);
 387
 388                                 if (pjcpu->target_freq > max_freq)
 389                                         max_freq = pjcpu->target_freq;
 390                         }
 391
 392                         if (max_freq != pcpu->policy->cur)
 393                                 __cpufreq_driver_target(pcpu->policy,
 394                                                         max_freq,
 395                                                         CPUFREQ_RELATION_H);
 396                         mutex_unlock(&set_speed_lock);
 397
 398                         pcpu->freq_change_time_in_idle =
 399                                 get_cpu_idle_time_us(cpu,
 400                                                      &pcpu->freq_change_time);
 401                         trace_cpufreq_interactive_up(cpu, pcpu->target_freq,
 402                                                      pcpu->policy->cur);
 403                 }
 404         }
 405
 406         return 0;
 407 }
 408
 409 static void cpufreq_interactive_freq_down(struct work_struct *work)
 410 {
 411         unsigned int cpu;
 412         cpumask_t tmp_mask;
 413         unsigned long flags;
 414         struct cpufreq_interactive_cpuinfo *pcpu;
 415
 416         spin_lock_irqsave(&down_cpumask_lock, flags);
 417         tmp_mask = down_cpumask;
 418         cpumask_clear(&down_cpumask);
 419         spin_unlock_irqrestore(&down_cpumask_lock, flags);
 420
 421         for_each_cpu(cpu, &tmp_mask) {
 422                 unsigned int j;
 423                 unsigned int max_freq = 0;
 424
 425                 pcpu = &per_cpu(cpuinfo, cpu);
 426                 smp_rmb();
 427
 428                 if (!pcpu->governor_enabled)
 429                         continue;
 430
 431                 mutex_lock(&set_speed_lock);
 432
 433                 for_each_cpu(j, pcpu->policy->cpus) {
 434                         struct cpufreq_interactive_cpuinfo *pjcpu =
 435                                 &per_cpu(cpuinfo, j);
 436
 437                         if (pjcpu->target_freq > max_freq)
 438                                 max_freq = pjcpu->target_freq;
 439                 }
 440
 441                 if (max_freq != pcpu->policy->cur)
 442                         __cpufreq_driver_target(pcpu->policy, max_freq,
 443                                                 CPUFREQ_RELATION_H);
 444
 445                 mutex_unlock(&set_speed_lock);
 446                 pcpu->freq_change_time_in_idle =
 447                         get_cpu_idle_time_us(cpu,
 448                                              &pcpu->freq_change_time);
 449                 trace_cpufreq_interactive_down(cpu, pcpu->target_freq,
 450                                                pcpu->policy->cur);
 451         }
 452 }
 453
 454 static ssize_t show_hispeed_freq(struct kobject *kobj,
 455                                  struct attribute *attr, char *buf)
 456 {
 457         return sprintf(buf, "%llu\n", hispeed_freq);
 458 }
 459
 460 static ssize_t store_hispeed_freq(struct kobject *kobj,
 461                                   struct attribute *attr, const char *buf,
 462                                   size_t count)
 463 {
 464         int ret;
 465         u64 val;
 466
 467         ret = strict_strtoull(buf, 0, &val);
 468         if (ret < 0)
 469                 return ret;
 470         hispeed_freq = val;
 471         return count;
 472 }
 473
 474 static struct global_attr hispeed_freq_attr = __ATTR(hispeed_freq, 0644,
 475                 show_hispeed_freq, store_hispeed_freq);
 476
 477
 478 static ssize_t show_go_hispeed_load(struct kobject *kobj,
 479                                      struct attribute *attr, char *buf)
 480 {
 481         return sprintf(buf, "%lu\n", go_hispeed_load);
 482 }
 483
 484 static ssize_t store_go_hispeed_load(struct kobject *kobj,
 485                         struct attribute *attr, const char *buf, size_t count)
 486 {
 487         int ret;
 488         unsigned long val;
 489
 490         ret = strict_strtoul(buf, 0, &val);
 491         if (ret < 0)
 492                 return ret;
 493         go_hispeed_load = val;
 494         return count;
 495 }
 496
 497 static struct global_attr go_hispeed_load_attr = __ATTR(go_hispeed_load, 0644,
 498                 show_go_hispeed_load, store_go_hispeed_load);
 499
 500 static ssize_t show_min_sample_time(struct kobject *kobj,
 501                                 struct attribute *attr, char *buf)
 502 {
 503         return sprintf(buf, "%lu\n", min_sample_time);
 504 }
 505
 506 static ssize_t store_min_sample_time(struct kobject *kobj,
 507                         struct attribute *attr, const char *buf, size_t count)
 508 {
 509         int ret;
 510         unsigned long val;
 511
 512         ret = strict_strtoul(buf, 0, &val);
 513         if (ret < 0)
 514                 return ret;
 515         min_sample_time = val;
 516         return count;
 517 }
 518
 519 static struct global_attr min_sample_time_attr = __ATTR(min_sample_time, 0644,
 520                 show_min_sample_time, store_min_sample_time);
 521
 522 static ssize_t show_timer_rate(struct kobject *kobj,
 523                         struct attribute *attr, char *buf)
 524 {
 525         return sprintf(buf, "%lu\n", timer_rate);
 526 }
 527
 528 static ssize_t store_timer_rate(struct kobject *kobj,
 529                         struct attribute *attr, const char *buf, size_t count)
 530 {
 531         int ret;
 532         unsigned long val;
 533
 534         ret = strict_strtoul(buf, 0, &val);
 535         if (ret < 0)
 536                 return ret;
 537         timer_rate = val;
 538         return count;
 539 }
 540
 541 static struct global_attr timer_rate_attr = __ATTR(timer_rate, 0644,
 542                 show_timer_rate, store_timer_rate);
 543
 544 static struct attribute *interactive_attributes[] = {
 545         &hispeed_freq_attr.attr,
 546         &go_hispeed_load_attr.attr,
 547         &min_sample_time_attr.attr,
 548         &timer_rate_attr.attr,
 549         NULL,
 550 };
 551
 552 static struct attribute_group interactive_attr_group = {
 553         .attrs = interactive_attributes,
 554         .name = "interactive",
 555 };
 556
 557 static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
 558                 unsigned int event)
 559 {
 560         int rc;
 561         unsigned int j;
 562         struct cpufreq_interactive_cpuinfo *pcpu;
 563         struct cpufreq_frequency_table *freq_table;
 564
 565         switch (event) {
 566         case CPUFREQ_GOV_START:
 567                 if (!cpu_online(policy->cpu))
 568                         return -EINVAL;
 569
 570                 freq_table =
 571                         cpufreq_frequency_get_table(policy->cpu);
 572
 573                 for_each_cpu(j, policy->cpus) {
 574                         pcpu = &per_cpu(cpuinfo, j);
 575                         pcpu->policy = policy;
 576                         pcpu->target_freq = policy->cur;
 577                         pcpu->freq_table = freq_table;
 578                         pcpu->freq_change_time_in_idle =
 579                                 get_cpu_idle_time_us(j,
 580                                              &pcpu->freq_change_time);
 581                         pcpu->governor_enabled = 1;
 582                         smp_wmb();
 583                 }
 584
 585                 if (!hispeed_freq)
 586                         hispeed_freq = policy->max;
 587
 588                 /*
 589                  * Do not register the idle hook and create sysfs
 590                  * entries if we have already done so.
 591                  */
 592                 if (atomic_inc_return(&active_count) > 1)
 593                         return 0;
 594
 595                 rc = sysfs_create_group(cpufreq_global_kobject,
 596                                 &interactive_attr_group);
 597                 if (rc)
 598                         return rc;
 599
 600                 break;
 601
 602         case CPUFREQ_GOV_STOP:
 603                 for_each_cpu(j, policy->cpus) {
 604                         pcpu = &per_cpu(cpuinfo, j);
 605                         pcpu->governor_enabled = 0;
 606                         smp_wmb();
 607                         del_timer_sync(&pcpu->cpu_timer);
 608
 609                         /*
 610                          * Reset idle exit time since we may cancel the timer
 611                          * before it can run after the last idle exit time,
 612                          * to avoid tripping the check in idle exit for a timer
 613                          * that is trying to run.
 614                          */
 615                         pcpu->idle_exit_time = 0;
 616                 }
 617
 618                 flush_work(&freq_scale_down_work);
 619                 if (atomic_dec_return(&active_count) > 0)
 620                         return 0;
 621
 622                 sysfs_remove_group(cpufreq_global_kobject,
 623                                 &interactive_attr_group);
 624
 625                 break;
 626
 627         case CPUFREQ_GOV_LIMITS:
 628                 if (policy->max < policy->cur)
 629                         __cpufreq_driver_target(policy,
 630                                         policy->max, CPUFREQ_RELATION_H);
 631                 else if (policy->min > policy->cur)
 632                         __cpufreq_driver_target(policy,
 633                                         policy->min, CPUFREQ_RELATION_L);
 634                 break;
 635         }
 636         return 0;
 637 }
 638
 639 static int cpufreq_interactive_idle_notifier(struct notifier_block *nb,
 640                                              unsigned long val,
 641                                              void *data)
 642 {
 643         switch (val) {
 644         case IDLE_START:
 645                 cpufreq_interactive_idle_start();
 646                 break;
 647         case IDLE_END:
 648                 cpufreq_interactive_idle_end();
 649                 break;
 650         }
 651
 652         return 0;
 653 }
 654
 655 static struct notifier_block cpufreq_interactive_idle_nb = {
 656         .notifier_call = cpufreq_interactive_idle_notifier,
 657 };
 658
 659 static int __init cpufreq_interactive_init(void)
 660 {
 661         unsigned int i;
 662         struct cpufreq_interactive_cpuinfo *pcpu;
 663         struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
 664
 665         go_hispeed_load = DEFAULT_GO_HISPEED_LOAD;
 666         min_sample_time = DEFAULT_MIN_SAMPLE_TIME;
 667         timer_rate = DEFAULT_TIMER_RATE;
 668
 669         /* Initalize per-cpu timers */
 670         for_each_possible_cpu(i) {
 671                 pcpu = &per_cpu(cpuinfo, i);
 672                 init_timer(&pcpu->cpu_timer);
 673                 pcpu->cpu_timer.function = cpufreq_interactive_timer;
 674                 pcpu->cpu_timer.data = i;
 675         }
 676
 677         up_task = kthread_create(cpufreq_interactive_up_task, NULL,
 678                                  "kinteractiveup");
 679         if (IS_ERR(up_task))
 680                 return PTR_ERR(up_task);
 681
 682         sched_setscheduler_nocheck(up_task, SCHED_FIFO, &param);
 683         get_task_struct(up_task);
 684
 685         /* No rescuer thread, bind to CPU queuing the work for possibly
 686            warm cache (probably doesn't matter much). */
 687         down_wq = alloc_workqueue("knteractive_down", 0, 1);
 688
 689         if (!down_wq)
 690                 goto err_freeuptask;
 691
 692         INIT_WORK(&freq_scale_down_work,
 693                   cpufreq_interactive_freq_down);
 694
 695         spin_lock_init(&up_cpumask_lock);
 696         spin_lock_init(&down_cpumask_lock);
 697         mutex_init(&set_speed_lock);
 698
 699         idle_notifier_register(&cpufreq_interactive_idle_nb);
 700
 701         return cpufreq_register_governor(&cpufreq_gov_interactive);
 702
 703 err_freeuptask:
 704         put_task_struct(up_task);
 705         return -ENOMEM;
 706 }
 707
 708 #ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE
 709 fs_initcall(cpufreq_interactive_init);
 710 #else
 711 module_init(cpufreq_interactive_init);
 712 #endif
 713
 714 static void __exit cpufreq_interactive_exit(void)
 715 {
 716         cpufreq_unregister_governor(&cpufreq_gov_interactive);
 717         kthread_stop(up_task);
 718         put_task_struct(up_task);
 719         destroy_workqueue(down_wq);
 720 }
 721
 722 module_exit(cpufreq_interactive_exit);
 723
 724 MODULE_AUTHOR("Mike Chan <mike@android.com>");
 725 MODULE_DESCRIPTION("'cpufreq_interactive' - A cpufreq governor for "
 726         "Latency sensitive workloads");
 727 MODULE_LICENSE("GPL");