drivers/cpufreq/cpufreq_interactive.c

   1 /*
   2  * drivers/cpufreq/cpufreq_interactive.c
   3  *
   4  * Copyright (C) 2010 Google, Inc.
   5  *
   6  * This software is licensed under the terms of the GNU General Public
   7  * License version 2, as published by the Free Software Foundation, and
   8  * may be copied, distributed, and modified under those terms.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * Author: Mike Chan (mike@android.com)
  16  *
  17  */
  18
  19 #include <linux/cpu.h>
  20 #include <linux/cpumask.h>
  21 #include <linux/cpufreq.h>
  22 #include <linux/module.h>
  23 #include <linux/mutex.h>
  24 #include <linux/sched.h>
  25 #include <linux/sched/rt.h>
  26 #include <linux/tick.h>
  27 #include <linux/time.h>
  28 #include <linux/timer.h>
  29 #include <linux/workqueue.h>
  30 #include <linux/kthread.h>
  31 #include <linux/mutex.h>
  32
  33 #define CREATE_TRACE_POINTS
  34 #include <trace/events/cpufreq_interactive.h>
  35
  36 #include <asm/cputime.h>
  37
  38 static atomic_t active_count = ATOMIC_INIT(0);
  39
  40 struct cpufreq_interactive_cpuinfo {
  41         struct timer_list cpu_timer;
  42         int timer_idlecancel;
  43         u64 time_in_idle;
  44         u64 idle_exit_time;
  45         u64 timer_run_time;
  46         int idling;
  47         u64 target_set_time;
  48         u64 target_set_time_in_idle;
  49         struct cpufreq_policy *policy;
  50         struct cpufreq_frequency_table *freq_table;
  51         unsigned int target_freq;
  52         int governor_enabled;
  53 };
  54
  55 static DEFINE_PER_CPU(struct cpufreq_interactive_cpuinfo, cpuinfo);
  56
  57 /* Workqueues handle frequency scaling */
  58 static struct task_struct *up_task;
  59 static struct workqueue_struct *down_wq;
  60 static struct work_struct freq_scale_down_work;
  61 static cpumask_t up_cpumask;
  62 static spinlock_t up_cpumask_lock;
  63 static cpumask_t down_cpumask;
  64 static spinlock_t down_cpumask_lock;
  65 static struct mutex set_speed_lock;
  66
  67 /* Hi speed to bump to from lo speed when load burst (default max) */
  68 static u64 hispeed_freq;
  69
  70 /* Go to hi speed when CPU load at or above this value. */
  71 #define DEFAULT_GO_HISPEED_LOAD 85
  72 static unsigned long go_hispeed_load;
  73
  74 /*
  75  * The minimum amount of time to spend at a frequency before we can ramp down.
  76  */
  77 #define DEFAULT_MIN_SAMPLE_TIME (80 * USEC_PER_MSEC)
  78 static unsigned long min_sample_time;
  79
  80 /*
  81  * The sample rate of the timer used to increase frequency
  82  */
  83 #define DEFAULT_TIMER_RATE (20 * USEC_PER_MSEC)
  84 static unsigned long timer_rate;
  85
  86 /*
  87  * Wait this long before raising speed above hispeed, by default a single
  88  * timer interval.
  89  */
  90 #define DEFAULT_ABOVE_HISPEED_DELAY DEFAULT_TIMER_RATE
  91 static unsigned long above_hispeed_delay_val;
  92
  93 static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
  94                 unsigned int event);
  95
  96 #ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE
  97 static
  98 #endif
  99 struct cpufreq_governor cpufreq_gov_interactive = {
 100         .name = "interactive",
 101         .governor = cpufreq_governor_interactive,
 102         .max_transition_latency = 10000000,
 103         .owner = THIS_MODULE,
 104 };
 105
 106 static void cpufreq_interactive_timer(unsigned long data)
 107 {
 108         unsigned int delta_idle;
 109         unsigned int delta_time;
 110         int cpu_load;
 111         int load_since_change;
 112         u64 time_in_idle;
 113         u64 idle_exit_time;
 114         struct cpufreq_interactive_cpuinfo *pcpu =
 115                 &per_cpu(cpuinfo, data);
 116         u64 now_idle;
 117         unsigned int new_freq;
 118         unsigned int index;
 119         unsigned long flags;
 120
 121         smp_rmb();
 122
 123         if (!pcpu->governor_enabled)
 124                 goto exit;
 125
 126         /*
 127          * Once pcpu->timer_run_time is updated to >= pcpu->idle_exit_time,
 128          * this lets idle exit know the current idle time sample has
 129          * been processed, and idle exit can generate a new sample and
 130          * re-arm the timer.  This prevents a concurrent idle
 131          * exit on that CPU from writing a new set of info at the same time
 132          * the timer function runs (the timer function can't use that info
 133          * until more time passes).
 134          */
 135         time_in_idle = pcpu->time_in_idle;
 136         idle_exit_time = pcpu->idle_exit_time;
 137         now_idle = get_cpu_idle_time_us(data, &pcpu->timer_run_time);
 138         smp_wmb();
 139
 140         /* If we raced with cancelling a timer, skip. */
 141         if (!idle_exit_time)
 142                 goto exit;
 143
 144         delta_idle = (unsigned int)(now_idle - time_in_idle);
 145         delta_time = (unsigned int)(pcpu->timer_run_time - idle_exit_time);
 146
 147         /*
 148          * If timer ran less than 1ms after short-term sample started, retry.
 149          */
 150         if (delta_time < 1000)
 151                 goto rearm;
 152
 153         if (delta_idle > delta_time)
 154                 cpu_load = 0;
 155         else
 156                 cpu_load = 100 * (delta_time - delta_idle) / delta_time;
 157
 158         delta_idle = (unsigned int)(now_idle - pcpu->target_set_time_in_idle);
 159         delta_time = (unsigned int)(pcpu->timer_run_time -
 160                                     pcpu->target_set_time);
 161
 162         if ((delta_time == 0) || (delta_idle > delta_time))
 163                 load_since_change = 0;
 164         else
 165                 load_since_change =
 166                         100 * (delta_time - delta_idle) / delta_time;
 167
 168         /*
 169          * Choose greater of short-term load (since last idle timer
 170          * started or timer function re-armed itself) or long-term load
 171          * (since last frequency change).
 172          */
 173         if (load_since_change > cpu_load)
 174                 cpu_load = load_since_change;
 175
 176         if (cpu_load >= go_hispeed_load) {
 177                 if (pcpu->target_freq <= pcpu->policy->min) {
 178                         new_freq = hispeed_freq;
 179                 } else {
 180                         new_freq = pcpu->policy->max * cpu_load / 100;
 181
 182                         if (new_freq < hispeed_freq)
 183                                 new_freq = hispeed_freq;
 184
 185                         if (pcpu->target_freq == hispeed_freq &&
 186                             new_freq > hispeed_freq &&
 187                             pcpu->timer_run_time - pcpu->target_set_time
 188                             < above_hispeed_delay_val) {
 189                                 trace_cpufreq_interactive_notyet(data, cpu_load,
 190                                                                  pcpu->target_freq,
 191                                                                  new_freq);
 192                                 goto rearm;
 193                         }
 194                 }
 195         } else {
 196                 new_freq = pcpu->policy->max * cpu_load / 100;
 197         }
 198
 199         if (cpufreq_frequency_table_target(pcpu->policy, pcpu->freq_table,
 200                                            new_freq, CPUFREQ_RELATION_H,
 201                                            &index)) {
 202                 pr_warn_once("timer %d: cpufreq_frequency_table_target error\n",
 203                              (int) data);
 204                 goto rearm;
 205         }
 206
 207         new_freq = pcpu->freq_table[index].frequency;
 208
 209         /*
 210          * Do not scale down unless we have been at this frequency for the
 211          * minimum sample time.
 212          */
 213         if (new_freq < pcpu->target_freq) {
 214                 if (pcpu->timer_run_time - pcpu->target_set_time
 215                     < min_sample_time) {
 216                         trace_cpufreq_interactive_notyet(data, cpu_load,
 217                                          pcpu->target_freq, new_freq);
 218                         goto rearm;
 219                 }
 220         }
 221
 222         pcpu->target_set_time_in_idle = now_idle;
 223         pcpu->target_set_time = pcpu->timer_run_time;
 224
 225         if (pcpu->target_freq == new_freq) {
 226                 trace_cpufreq_interactive_already(data, cpu_load,
 227                                                   pcpu->target_freq, new_freq);
 228                 goto rearm_if_notmax;
 229         }
 230
 231         trace_cpufreq_interactive_target(data, cpu_load, pcpu->target_freq,
 232                                          new_freq);
 233
 234         if (new_freq < pcpu->target_freq) {
 235                 pcpu->target_freq = new_freq;
 236                 spin_lock_irqsave(&down_cpumask_lock, flags);
 237                 cpumask_set_cpu(data, &down_cpumask);
 238                 spin_unlock_irqrestore(&down_cpumask_lock, flags);
 239                 queue_work(down_wq, &freq_scale_down_work);
 240         } else {
 241                 pcpu->target_freq = new_freq;
 242                 spin_lock_irqsave(&up_cpumask_lock, flags);
 243                 cpumask_set_cpu(data, &up_cpumask);
 244                 spin_unlock_irqrestore(&up_cpumask_lock, flags);
 245                 wake_up_process(up_task);
 246         }
 247
 248 rearm_if_notmax:
 249         /*
 250          * Already set max speed and don't see a need to change that,
 251          * wait until next idle to re-evaluate, don't need timer.
 252          */
 253         if (pcpu->target_freq == pcpu->policy->max)
 254                 goto exit;
 255
 256 rearm:
 257         if (!timer_pending(&pcpu->cpu_timer)) {
 258                 /*
 259                  * If already at min: if that CPU is idle, don't set timer.
 260                  * Else cancel the timer if that CPU goes idle.  We don't
 261                  * need to re-evaluate speed until the next idle exit.
 262                  */
 263                 if (pcpu->target_freq == pcpu->policy->min) {
 264                         smp_rmb();
 265
 266                         if (pcpu->idling)
 267                                 goto exit;
 268
 269                         pcpu->timer_idlecancel = 1;
 270                 }
 271
 272                 pcpu->time_in_idle = get_cpu_idle_time_us(
 273                         data, &pcpu->idle_exit_time);
 274                 mod_timer(&pcpu->cpu_timer,
 275                           jiffies + usecs_to_jiffies(timer_rate));
 276         }
 277
 278 exit:
 279         return;
 280 }
 281
 282 static void cpufreq_interactive_idle_start(void)
 283 {
 284         struct cpufreq_interactive_cpuinfo *pcpu =
 285                 &per_cpu(cpuinfo, smp_processor_id());
 286         int pending;
 287
 288         if (!pcpu->governor_enabled)
 289                 return;
 290
 291         pcpu->idling = 1;
 292         smp_wmb();
 293         pending = timer_pending(&pcpu->cpu_timer);
 294
 295         if (pcpu->target_freq != pcpu->policy->min) {
 296 #ifdef CONFIG_SMP
 297                 /*
 298                  * Entering idle while not at lowest speed.  On some
 299                  * platforms this can hold the other CPU(s) at that speed
 300                  * even though the CPU is idle. Set a timer to re-evaluate
 301                  * speed so this idle CPU doesn't hold the other CPUs above
 302                  * min indefinitely.  This should probably be a quirk of
 303                  * the CPUFreq driver.
 304                  */
 305                 if (!pending) {
 306                         pcpu->time_in_idle = get_cpu_idle_time_us(
 307                                 smp_processor_id(), &pcpu->idle_exit_time);
 308                         pcpu->timer_idlecancel = 0;
 309                         mod_timer(&pcpu->cpu_timer,
 310                                   jiffies + usecs_to_jiffies(timer_rate));
 311                 }
 312 #endif
 313         } else {
 314                 /*
 315                  * If at min speed and entering idle after load has
 316                  * already been evaluated, and a timer has been set just in
 317                  * case the CPU suddenly goes busy, cancel that timer.  The
 318                  * CPU didn't go busy; we'll recheck things upon idle exit.
 319                  */
 320                 if (pending && pcpu->timer_idlecancel) {
 321                         del_timer(&pcpu->cpu_timer);
 322                         /*
 323                          * Ensure last timer run time is after current idle
 324                          * sample start time, so next idle exit will always
 325                          * start a new idle sampling period.
 326                          */
 327                         pcpu->idle_exit_time = 0;
 328                         pcpu->timer_idlecancel = 0;
 329                 }
 330         }
 331
 332 }
 333
 334 static void cpufreq_interactive_idle_end(void)
 335 {
 336         struct cpufreq_interactive_cpuinfo *pcpu =
 337                 &per_cpu(cpuinfo, smp_processor_id());
 338
 339         pcpu->idling = 0;
 340         smp_wmb();
 341
 342         /*
 343          * Arm the timer for 1-2 ticks later if not already, and if the timer
 344          * function has already processed the previous load sampling
 345          * interval.  (If the timer is not pending but has not processed
 346          * the previous interval, it is probably racing with us on another
 347          * CPU.  Let it compute load based on the previous sample and then
 348          * re-arm the timer for another interval when it's done, rather
 349          * than updating the interval start time to be "now", which doesn't
 350          * give the timer function enough time to make a decision on this
 351          * run.)
 352          */
 353         if (timer_pending(&pcpu->cpu_timer) == 0 &&
 354             pcpu->timer_run_time >= pcpu->idle_exit_time &&
 355             pcpu->governor_enabled) {
 356                 pcpu->time_in_idle =
 357                         get_cpu_idle_time_us(smp_processor_id(),
 358                                              &pcpu->idle_exit_time);
 359                 pcpu->timer_idlecancel = 0;
 360                 mod_timer(&pcpu->cpu_timer,
 361                           jiffies + usecs_to_jiffies(timer_rate));
 362         }
 363
 364 }
 365
 366 static int cpufreq_interactive_up_task(void *data)
 367 {
 368         unsigned int cpu;
 369         cpumask_t tmp_mask;
 370         unsigned long flags;
 371         struct cpufreq_interactive_cpuinfo *pcpu;
 372
 373         while (1) {
 374                 set_current_state(TASK_INTERRUPTIBLE);
 375                 spin_lock_irqsave(&up_cpumask_lock, flags);
 376
 377                 if (cpumask_empty(&up_cpumask)) {
 378                         spin_unlock_irqrestore(&up_cpumask_lock, flags);
 379                         schedule();
 380
 381                         if (kthread_should_stop())
 382                                 break;
 383
 384                         spin_lock_irqsave(&up_cpumask_lock, flags);
 385                 }
 386
 387                 set_current_state(TASK_RUNNING);
 388                 tmp_mask = up_cpumask;
 389                 cpumask_clear(&up_cpumask);
 390                 spin_unlock_irqrestore(&up_cpumask_lock, flags);
 391
 392                 for_each_cpu(cpu, &tmp_mask) {
 393                         unsigned int j;
 394                         unsigned int max_freq = 0;
 395
 396                         pcpu = &per_cpu(cpuinfo, cpu);
 397                         smp_rmb();
 398
 399                         if (!pcpu->governor_enabled)
 400                                 continue;
 401
 402                         mutex_lock(&set_speed_lock);
 403
 404                         for_each_cpu(j, pcpu->policy->cpus) {
 405                                 struct cpufreq_interactive_cpuinfo *pjcpu =
 406                                         &per_cpu(cpuinfo, j);
 407
 408                                 if (pjcpu->target_freq > max_freq)
 409                                         max_freq = pjcpu->target_freq;
 410                         }
 411
 412                         if (max_freq != pcpu->policy->cur)
 413                                 __cpufreq_driver_target(pcpu->policy,
 414                                                         max_freq,
 415                                                         CPUFREQ_RELATION_H);
 416                         mutex_unlock(&set_speed_lock);
 417                         trace_cpufreq_interactive_up(cpu, pcpu->target_freq,
 418                                                      pcpu->policy->cur);
 419                 }
 420         }
 421
 422         return 0;
 423 }
 424
 425 static void cpufreq_interactive_freq_down(struct work_struct *work)
 426 {
 427         unsigned int cpu;
 428         cpumask_t tmp_mask;
 429         unsigned long flags;
 430         struct cpufreq_interactive_cpuinfo *pcpu;
 431
 432         spin_lock_irqsave(&down_cpumask_lock, flags);
 433         tmp_mask = down_cpumask;
 434         cpumask_clear(&down_cpumask);
 435         spin_unlock_irqrestore(&down_cpumask_lock, flags);
 436
 437         for_each_cpu(cpu, &tmp_mask) {
 438                 unsigned int j;
 439                 unsigned int max_freq = 0;
 440
 441                 pcpu = &per_cpu(cpuinfo, cpu);
 442                 smp_rmb();
 443
 444                 if (!pcpu->governor_enabled)
 445                         continue;
 446
 447                 mutex_lock(&set_speed_lock);
 448
 449                 for_each_cpu(j, pcpu->policy->cpus) {
 450                         struct cpufreq_interactive_cpuinfo *pjcpu =
 451                                 &per_cpu(cpuinfo, j);
 452
 453                         if (pjcpu->target_freq > max_freq)
 454                                 max_freq = pjcpu->target_freq;
 455                 }
 456
 457                 if (max_freq != pcpu->policy->cur)
 458                         __cpufreq_driver_target(pcpu->policy, max_freq,
 459                                                 CPUFREQ_RELATION_H);
 460
 461                 mutex_unlock(&set_speed_lock);
 462                 trace_cpufreq_interactive_down(cpu, pcpu->target_freq,
 463                                                pcpu->policy->cur);
 464         }
 465 }
 466
 467 static ssize_t show_hispeed_freq(struct kobject *kobj,
 468                                  struct attribute *attr, char *buf)
 469 {
 470         return sprintf(buf, "%llu\n", hispeed_freq);
 471 }
 472
 473 static ssize_t store_hispeed_freq(struct kobject *kobj,
 474                                   struct attribute *attr, const char *buf,
 475                                   size_t count)
 476 {
 477         int ret;
 478         u64 val;
 479
 480         ret = strict_strtoull(buf, 0, &val);
 481         if (ret < 0)
 482                 return ret;
 483         hispeed_freq = val;
 484         return count;
 485 }
 486
 487 static struct global_attr hispeed_freq_attr = __ATTR(hispeed_freq, 0644,
 488                 show_hispeed_freq, store_hispeed_freq);
 489
 490
 491 static ssize_t show_go_hispeed_load(struct kobject *kobj,
 492                                      struct attribute *attr, char *buf)
 493 {
 494         return sprintf(buf, "%lu\n", go_hispeed_load);
 495 }
 496
 497 static ssize_t store_go_hispeed_load(struct kobject *kobj,
 498                         struct attribute *attr, const char *buf, size_t count)
 499 {
 500         int ret;
 501         unsigned long val;
 502
 503         ret = strict_strtoul(buf, 0, &val);
 504         if (ret < 0)
 505                 return ret;
 506         go_hispeed_load = val;
 507         return count;
 508 }
 509
 510 static struct global_attr go_hispeed_load_attr = __ATTR(go_hispeed_load, 0644,
 511                 show_go_hispeed_load, store_go_hispeed_load);
 512
 513 static ssize_t show_min_sample_time(struct kobject *kobj,
 514                                 struct attribute *attr, char *buf)
 515 {
 516         return sprintf(buf, "%lu\n", min_sample_time);
 517 }
 518
 519 static ssize_t store_min_sample_time(struct kobject *kobj,
 520                         struct attribute *attr, const char *buf, size_t count)
 521 {
 522         int ret;
 523         unsigned long val;
 524
 525         ret = strict_strtoul(buf, 0, &val);
 526         if (ret < 0)
 527                 return ret;
 528         min_sample_time = val;
 529         return count;
 530 }
 531
 532 static struct global_attr min_sample_time_attr = __ATTR(min_sample_time, 0644,
 533                 show_min_sample_time, store_min_sample_time);
 534
 535 static ssize_t show_above_hispeed_delay(struct kobject *kobj,
 536                                         struct attribute *attr, char *buf)
 537 {
 538         return sprintf(buf, "%lu\n", above_hispeed_delay_val);
 539 }
 540
 541 static ssize_t store_above_hispeed_delay(struct kobject *kobj,
 542                                          struct attribute *attr,
 543                                          const char *buf, size_t count)
 544 {
 545         int ret;
 546         unsigned long val;
 547
 548         ret = strict_strtoul(buf, 0, &val);
 549         if (ret < 0)
 550                 return ret;
 551         above_hispeed_delay_val = val;
 552         return count;
 553 }
 554
 555 define_one_global_rw(above_hispeed_delay);
 556
 557 static ssize_t show_timer_rate(struct kobject *kobj,
 558                         struct attribute *attr, char *buf)
 559 {
 560         return sprintf(buf, "%lu\n", timer_rate);
 561 }
 562
 563 static ssize_t store_timer_rate(struct kobject *kobj,
 564                         struct attribute *attr, const char *buf, size_t count)
 565 {
 566         int ret;
 567         unsigned long val;
 568
 569         ret = strict_strtoul(buf, 0, &val);
 570         if (ret < 0)
 571                 return ret;
 572         timer_rate = val;
 573         return count;
 574 }
 575
 576 static struct global_attr timer_rate_attr = __ATTR(timer_rate, 0644,
 577                 show_timer_rate, store_timer_rate);
 578
 579 static struct attribute *interactive_attributes[] = {
 580         &hispeed_freq_attr.attr,
 581         &go_hispeed_load_attr.attr,
 582         &above_hispeed_delay.attr,
 583         &min_sample_time_attr.attr,
 584         &timer_rate_attr.attr,
 585         NULL,
 586 };
 587
 588 static struct attribute_group interactive_attr_group = {
 589         .attrs = interactive_attributes,
 590         .name = "interactive",
 591 };
 592
 593 static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
 594                 unsigned int event)
 595 {
 596         int rc;
 597         unsigned int j;
 598         struct cpufreq_interactive_cpuinfo *pcpu;
 599         struct cpufreq_frequency_table *freq_table;
 600
 601         switch (event) {
 602         case CPUFREQ_GOV_START:
 603                 if (!cpu_online(policy->cpu))
 604                         return -EINVAL;
 605
 606                 freq_table =
 607                         cpufreq_frequency_get_table(policy->cpu);
 608
 609                 for_each_cpu(j, policy->cpus) {
 610                         pcpu = &per_cpu(cpuinfo, j);
 611                         pcpu->policy = policy;
 612                         pcpu->target_freq = policy->cur;
 613                         pcpu->freq_table = freq_table;
 614                         pcpu->target_set_time_in_idle =
 615                                 get_cpu_idle_time_us(j,
 616                                              &pcpu->target_set_time);
 617                         pcpu->governor_enabled = 1;
 618                         smp_wmb();
 619                 }
 620
 621                 if (!hispeed_freq)
 622                         hispeed_freq = policy->max;
 623
 624                 /*
 625                  * Do not register the idle hook and create sysfs
 626                  * entries if we have already done so.
 627                  */
 628                 if (atomic_inc_return(&active_count) > 1)
 629                         return 0;
 630
 631                 rc = sysfs_create_group(cpufreq_global_kobject,
 632                                 &interactive_attr_group);
 633                 if (rc)
 634                         return rc;
 635
 636                 break;
 637
 638         case CPUFREQ_GOV_STOP:
 639                 for_each_cpu(j, policy->cpus) {
 640                         pcpu = &per_cpu(cpuinfo, j);
 641                         pcpu->governor_enabled = 0;
 642                         smp_wmb();
 643                         del_timer_sync(&pcpu->cpu_timer);
 644
 645                         /*
 646                          * Reset idle exit time since we may cancel the timer
 647                          * before it can run after the last idle exit time,
 648                          * to avoid tripping the check in idle exit for a timer
 649                          * that is trying to run.
 650                          */
 651                         pcpu->idle_exit_time = 0;
 652                 }
 653
 654                 flush_work(&freq_scale_down_work);
 655                 if (atomic_dec_return(&active_count) > 0)
 656                         return 0;
 657
 658                 sysfs_remove_group(cpufreq_global_kobject,
 659                                 &interactive_attr_group);
 660
 661                 break;
 662
 663         case CPUFREQ_GOV_LIMITS:
 664                 if (policy->max < policy->cur)
 665                         __cpufreq_driver_target(policy,
 666                                         policy->max, CPUFREQ_RELATION_H);
 667                 else if (policy->min > policy->cur)
 668                         __cpufreq_driver_target(policy,
 669                                         policy->min, CPUFREQ_RELATION_L);
 670                 break;
 671         }
 672         return 0;
 673 }
 674
 675 static int cpufreq_interactive_idle_notifier(struct notifier_block *nb,
 676                                              unsigned long val,
 677                                              void *data)
 678 {
 679         switch (val) {
 680         case IDLE_START:
 681                 cpufreq_interactive_idle_start();
 682                 break;
 683         case IDLE_END:
 684                 cpufreq_interactive_idle_end();
 685                 break;
 686         }
 687
 688         return 0;
 689 }
 690
 691 static struct notifier_block cpufreq_interactive_idle_nb = {
 692         .notifier_call = cpufreq_interactive_idle_notifier,
 693 };
 694
 695 static int __init cpufreq_interactive_init(void)
 696 {
 697         unsigned int i;
 698         struct cpufreq_interactive_cpuinfo *pcpu;
 699         struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
 700
 701         go_hispeed_load = DEFAULT_GO_HISPEED_LOAD;
 702         min_sample_time = DEFAULT_MIN_SAMPLE_TIME;
 703         above_hispeed_delay_val = DEFAULT_ABOVE_HISPEED_DELAY;
 704         timer_rate = DEFAULT_TIMER_RATE;
 705
 706         /* Initalize per-cpu timers */
 707         for_each_possible_cpu(i) {
 708                 pcpu = &per_cpu(cpuinfo, i);
 709                 init_timer(&pcpu->cpu_timer);
 710                 pcpu->cpu_timer.function = cpufreq_interactive_timer;
 711                 pcpu->cpu_timer.data = i;
 712         }
 713
 714         up_task = kthread_create(cpufreq_interactive_up_task, NULL,
 715                                  "kinteractiveup");
 716         if (IS_ERR(up_task))
 717                 return PTR_ERR(up_task);
 718
 719         sched_setscheduler_nocheck(up_task, SCHED_FIFO, &param);
 720         get_task_struct(up_task);
 721
 722         /* No rescuer thread, bind to CPU queuing the work for possibly
 723            warm cache (probably doesn't matter much). */
 724         down_wq = alloc_workqueue("knteractive_down", 0, 1);
 725
 726         if (!down_wq)
 727                 goto err_freeuptask;
 728
 729         INIT_WORK(&freq_scale_down_work,
 730                   cpufreq_interactive_freq_down);
 731
 732         spin_lock_init(&up_cpumask_lock);
 733         spin_lock_init(&down_cpumask_lock);
 734         mutex_init(&set_speed_lock);
 735
 736         idle_notifier_register(&cpufreq_interactive_idle_nb);
 737
 738         return cpufreq_register_governor(&cpufreq_gov_interactive);
 739
 740 err_freeuptask:
 741         put_task_struct(up_task);
 742         return -ENOMEM;
 743 }
 744
 745 #ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE
 746 fs_initcall(cpufreq_interactive_init);
 747 #else
 748 module_init(cpufreq_interactive_init);
 749 #endif
 750
 751 static void __exit cpufreq_interactive_exit(void)
 752 {
 753         cpufreq_unregister_governor(&cpufreq_gov_interactive);
 754         kthread_stop(up_task);
 755         put_task_struct(up_task);
 756         destroy_workqueue(down_wq);
 757 }
 758
 759 module_exit(cpufreq_interactive_exit);
 760
 761 MODULE_AUTHOR("Mike Chan <mike@android.com>");
 762 MODULE_DESCRIPTION("'cpufreq_interactive' - A cpufreq governor for "
 763         "Latency sensitive workloads");
 764 MODULE_LICENSE("GPL");