drivers/cpufreq/cpufreq_interactive.c

   1 /*
   2  * drivers/cpufreq/cpufreq_interactive.c
   3  *
   4  * Copyright (C) 2010 Google, Inc.
   5  *
   6  * This software is licensed under the terms of the GNU General Public
   7  * License version 2, as published by the Free Software Foundation, and
   8  * may be copied, distributed, and modified under those terms.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * Author: Mike Chan (mike@android.com)
  16  *
  17  */
  18
  19 #include <linux/cpu.h>
  20 #include <linux/cpumask.h>
  21 #include <linux/cpufreq.h>
  22 #include <linux/module.h>
  23 #include <linux/moduleparam.h>
  24 #include <linux/mutex.h>
  25 #include <linux/sched.h>
  26 #include <linux/sched/rt.h>
  27 #include <linux/tick.h>
  28 #include <linux/time.h>
  29 #include <linux/timer.h>
  30 #include <linux/workqueue.h>
  31 #include <linux/kthread.h>
  32 #include <linux/mutex.h>
  33 #include <linux/slab.h>
  34 #include <asm/cputime.h>
  35
  36 #define CREATE_TRACE_POINTS
  37 #include <trace/events/cpufreq_interactive.h>
  38
  39 static atomic_t active_count = ATOMIC_INIT(0);
  40
  41 struct cpufreq_interactive_cpuinfo {
  42         struct timer_list cpu_timer;
  43         int timer_idlecancel;
  44         u64 time_in_idle;
  45         u64 time_in_idle_timestamp;
  46         u64 target_set_time;
  47         u64 target_set_time_in_idle;
  48         struct cpufreq_policy *policy;
  49         struct cpufreq_frequency_table *freq_table;
  50         unsigned int target_freq;
  51         unsigned int floor_freq;
  52         u64 floor_validate_time;
  53         u64 hispeed_validate_time;
  54         int governor_enabled;
  55 };
  56
  57 static DEFINE_PER_CPU(struct cpufreq_interactive_cpuinfo, cpuinfo);
  58
  59 /* realtime thread handles frequency scaling */
  60 static struct task_struct *speedchange_task;
  61 static cpumask_t speedchange_cpumask;
  62 static spinlock_t speedchange_cpumask_lock;
  63
  64 /* Hi speed to bump to from lo speed when load burst (default max) */
  65 static unsigned int hispeed_freq;
  66
  67 /* Go to hi speed when CPU load at or above this value. */
  68 #define DEFAULT_GO_HISPEED_LOAD 85
  69 static unsigned long go_hispeed_load;
  70
  71 /* Target load.  Lower values result in higher CPU speeds. */
  72 #define DEFAULT_TARGET_LOAD 90
  73 static unsigned int default_target_loads[] = {DEFAULT_TARGET_LOAD};
  74 static spinlock_t target_loads_lock;
  75 static unsigned int *target_loads = default_target_loads;
  76 static int ntarget_loads = ARRAY_SIZE(default_target_loads);
  77
  78 /*
  79  * The minimum amount of time to spend at a frequency before we can ramp down.
  80  */
  81 #define DEFAULT_MIN_SAMPLE_TIME (80 * USEC_PER_MSEC)
  82 static unsigned long min_sample_time;
  83
  84 /*
  85  * The sample rate of the timer used to increase frequency
  86  */
  87 #define DEFAULT_TIMER_RATE (20 * USEC_PER_MSEC)
  88 static unsigned long timer_rate;
  89
  90 /*
  91  * Wait this long before raising speed above hispeed, by default a single
  92  * timer interval.
  93  */
  94 #define DEFAULT_ABOVE_HISPEED_DELAY DEFAULT_TIMER_RATE
  95 static unsigned long above_hispeed_delay_val;
  96
  97 /*
  98  * Non-zero means longer-term speed boost active.
  99  */
 100
 101 static int boost_val;
 102
 103 static bool governidle;
 104 module_param(governidle, bool, S_IWUSR | S_IRUGO);
 105 MODULE_PARM_DESC(governidle,
 106         "Set to 1 to wake up CPUs from idle to reduce speed (default 0)");
 107
 108 static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
 109                 unsigned int event);
 110
 111 #ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE
 112 static
 113 #endif
 114 struct cpufreq_governor cpufreq_gov_interactive = {
 115         .name = "interactive",
 116         .governor = cpufreq_governor_interactive,
 117         .max_transition_latency = 10000000,
 118         .owner = THIS_MODULE,
 119 };
 120
 121 static void cpufreq_interactive_timer_resched(
 122         struct cpufreq_interactive_cpuinfo *pcpu)
 123 {
 124         mod_timer_pinned(&pcpu->cpu_timer,
 125                          jiffies + usecs_to_jiffies(timer_rate));
 126         pcpu->time_in_idle =
 127                 get_cpu_idle_time_us(smp_processor_id(),
 128                                      &pcpu->time_in_idle_timestamp);
 129 }
 130
 131 static unsigned int freq_to_targetload(unsigned int freq)
 132 {
 133         int i;
 134         unsigned int ret;
 135
 136         spin_lock(&target_loads_lock);
 137
 138         for (i = 0; i < ntarget_loads - 1 && freq >= target_loads[i+1]; i += 2)
 139                 ;
 140
 141         ret = target_loads[i];
 142         spin_unlock(&target_loads_lock);
 143         return ret;
 144 }
 145
 146 /*
 147  * If increasing frequencies never map to a lower target load then
 148  * choose_freq() will find the minimum frequency that does not exceed its
 149  * target load given the current load.
 150  */
 151
 152 static unsigned int choose_freq(
 153         struct cpufreq_interactive_cpuinfo *pcpu, unsigned int curload)
 154 {
 155         unsigned int freq = pcpu->policy->cur;
 156         unsigned int loadadjfreq = freq * curload;
 157         unsigned int prevfreq, freqmin, freqmax;
 158         unsigned int tl;
 159         int index;
 160
 161         freqmin = 0;
 162         freqmax = UINT_MAX;
 163
 164         do {
 165                 prevfreq = freq;
 166                 tl = freq_to_targetload(freq);
 167
 168                 /*
 169                  * Find the lowest frequency where the computed load is less
 170                  * than or equal to the target load.
 171                  */
 172
 173                 cpufreq_frequency_table_target(
 174                         pcpu->policy, pcpu->freq_table, loadadjfreq / tl,
 175                         CPUFREQ_RELATION_L, &index);
 176                 freq = pcpu->freq_table[index].frequency;
 177
 178                 if (freq > prevfreq) {
 179                         /* The previous frequency is too low. */
 180                         freqmin = prevfreq;
 181
 182                         if (freq >= freqmax) {
 183                                 /*
 184                                  * Find the highest frequency that is less
 185                                  * than freqmax.
 186                                  */
 187                                 cpufreq_frequency_table_target(
 188                                         pcpu->policy, pcpu->freq_table,
 189                                         freqmax - 1, CPUFREQ_RELATION_H,
 190                                         &index);
 191                                 freq = pcpu->freq_table[index].frequency;
 192
 193                                 if (freq == freqmin) {
 194                                         /*
 195                                          * The first frequency below freqmax
 196                                          * has already been found to be too
 197                                          * low.  freqmax is the lowest speed
 198                                          * we found that is fast enough.
 199                                          */
 200                                         freq = freqmax;
 201                                         break;
 202                                 }
 203                         }
 204                 } else if (freq < prevfreq) {
 205                         /* The previous frequency is high enough. */
 206                         freqmax = prevfreq;
 207
 208                         if (freq <= freqmin) {
 209                                 /*
 210                                  * Find the lowest frequency that is higher
 211                                  * than freqmin.
 212                                  */
 213                                 cpufreq_frequency_table_target(
 214                                         pcpu->policy, pcpu->freq_table,
 215                                         freqmin + 1, CPUFREQ_RELATION_L,
 216                                         &index);
 217                                 freq = pcpu->freq_table[index].frequency;
 218
 219                                 /*
 220                                  * If freqmax is the first frequency above
 221                                  * freqmin then we have already found that
 222                                  * this speed is fast enough.
 223                                  */
 224                                 if (freq == freqmax)
 225                                         break;
 226                         }
 227                 }
 228
 229                 /* If same frequency chosen as previous then done. */
 230         } while (freq != prevfreq);
 231
 232         return freq;
 233 }
 234
 235 static void cpufreq_interactive_timer(unsigned long data)
 236 {
 237         u64 now;
 238         unsigned int delta_idle;
 239         unsigned int delta_time;
 240         int cpu_load;
 241         int load_since_change;
 242         struct cpufreq_interactive_cpuinfo *pcpu =
 243                 &per_cpu(cpuinfo, data);
 244         u64 now_idle;
 245         unsigned int new_freq;
 246         unsigned int index;
 247         unsigned long flags;
 248
 249         smp_rmb();
 250
 251         if (!pcpu->governor_enabled)
 252                 goto exit;
 253
 254         now_idle = get_cpu_idle_time_us(data, &now);
 255         delta_idle = (unsigned int)(now_idle - pcpu->time_in_idle);
 256         delta_time = (unsigned int)(now - pcpu->time_in_idle_timestamp);
 257
 258         /*
 259          * If timer ran less than 1ms after short-term sample started, retry.
 260          */
 261         if (delta_time < 1000)
 262                 goto rearm;
 263
 264         if (delta_idle > delta_time)
 265                 cpu_load = 0;
 266         else
 267                 cpu_load = 100 * (delta_time - delta_idle) / delta_time;
 268
 269         delta_idle = (unsigned int)(now_idle - pcpu->target_set_time_in_idle);
 270         delta_time = (unsigned int)(now - pcpu->target_set_time);
 271
 272         if ((delta_time == 0) || (delta_idle > delta_time))
 273                 load_since_change = 0;
 274         else
 275                 load_since_change =
 276                         100 * (delta_time - delta_idle) / delta_time;
 277
 278         /*
 279          * Choose greater of short-term load (since last idle timer
 280          * started or timer function re-armed itself) or long-term load
 281          * (since last frequency change).
 282          */
 283         if (load_since_change > cpu_load)
 284                 cpu_load = load_since_change;
 285
 286         if ((cpu_load >= go_hispeed_load || boost_val) &&
 287             pcpu->target_freq < hispeed_freq)
 288                 new_freq = hispeed_freq;
 289         else
 290                 new_freq = choose_freq(pcpu, cpu_load);
 291
 292         if (pcpu->target_freq >= hispeed_freq &&
 293             new_freq > pcpu->target_freq &&
 294             now - pcpu->hispeed_validate_time < above_hispeed_delay_val) {
 295                 trace_cpufreq_interactive_notyet(
 296                         data, cpu_load, pcpu->target_freq,
 297                         pcpu->policy->cur, new_freq);
 298                 goto rearm;
 299         }
 300
 301         pcpu->hispeed_validate_time = now;
 302
 303         if (cpufreq_frequency_table_target(pcpu->policy, pcpu->freq_table,
 304                                            new_freq, CPUFREQ_RELATION_L,
 305                                            &index)) {
 306                 pr_warn_once("timer %d: cpufreq_frequency_table_target error\n",
 307                              (int) data);
 308                 goto rearm;
 309         }
 310
 311         new_freq = pcpu->freq_table[index].frequency;
 312
 313         /*
 314          * Do not scale below floor_freq unless we have been at or above the
 315          * floor frequency for the minimum sample time since last validated.
 316          */
 317         if (new_freq < pcpu->floor_freq) {
 318                 if (now - pcpu->floor_validate_time < min_sample_time) {
 319                         trace_cpufreq_interactive_notyet(
 320                                 data, cpu_load, pcpu->target_freq,
 321                                 pcpu->policy->cur, new_freq);
 322                         goto rearm;
 323                 }
 324         }
 325
 326         pcpu->floor_freq = new_freq;
 327         pcpu->floor_validate_time = now;
 328
 329         if (pcpu->target_freq == new_freq) {
 330                 trace_cpufreq_interactive_already(
 331                         data, cpu_load, pcpu->target_freq,
 332                         pcpu->policy->cur, new_freq);
 333                 goto rearm_if_notmax;
 334         }
 335
 336         trace_cpufreq_interactive_target(data, cpu_load, pcpu->target_freq,
 337                                          pcpu->policy->cur, new_freq);
 338         pcpu->target_set_time_in_idle = now_idle;
 339         pcpu->target_set_time = now;
 340
 341         pcpu->target_freq = new_freq;
 342         spin_lock_irqsave(&speedchange_cpumask_lock, flags);
 343         cpumask_set_cpu(data, &speedchange_cpumask);
 344         spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);
 345         wake_up_process(speedchange_task);
 346
 347 rearm_if_notmax:
 348         /*
 349          * Already set max speed and don't see a need to change that,
 350          * wait until next idle to re-evaluate, don't need timer.
 351          */
 352         if (pcpu->target_freq == pcpu->policy->max)
 353                 goto exit;
 354
 355 rearm:
 356         if (!timer_pending(&pcpu->cpu_timer)) {
 357                 /*
 358                  * If governing speed in idle and already at min, cancel the
 359                  * timer if that CPU goes idle.  We don't need to re-evaluate
 360                  * speed until the next idle exit.
 361                  */
 362                 if (governidle && pcpu->target_freq == pcpu->policy->min)
 363                         pcpu->timer_idlecancel = 1;
 364
 365                 cpufreq_interactive_timer_resched(pcpu);
 366         }
 367
 368 exit:
 369         return;
 370 }
 371
 372 static void cpufreq_interactive_idle_start(void)
 373 {
 374         struct cpufreq_interactive_cpuinfo *pcpu =
 375                 &per_cpu(cpuinfo, smp_processor_id());
 376         int pending;
 377
 378         if (!pcpu->governor_enabled)
 379                 return;
 380
 381         pending = timer_pending(&pcpu->cpu_timer);
 382
 383         if (pcpu->target_freq != pcpu->policy->min) {
 384                 /*
 385                  * Entering idle while not at lowest speed.  On some
 386                  * platforms this can hold the other CPU(s) at that speed
 387                  * even though the CPU is idle. Set a timer to re-evaluate
 388                  * speed so this idle CPU doesn't hold the other CPUs above
 389                  * min indefinitely.  This should probably be a quirk of
 390                  * the CPUFreq driver.
 391                  */
 392                 if (!pending) {
 393                         pcpu->timer_idlecancel = 0;
 394                         cpufreq_interactive_timer_resched(pcpu);
 395                 }
 396         } else if (governidle) {
 397                 /*
 398                  * If at min speed and entering idle after load has
 399                  * already been evaluated, and a timer has been set just in
 400                  * case the CPU suddenly goes busy, cancel that timer.  The
 401                  * CPU didn't go busy; we'll recheck things upon idle exit.
 402                  */
 403                 if (pending && pcpu->timer_idlecancel) {
 404                         del_timer(&pcpu->cpu_timer);
 405                         pcpu->timer_idlecancel = 0;
 406                 }
 407         }
 408
 409 }
 410
 411 static void cpufreq_interactive_idle_end(void)
 412 {
 413         struct cpufreq_interactive_cpuinfo *pcpu =
 414                 &per_cpu(cpuinfo, smp_processor_id());
 415
 416         if (!pcpu->governor_enabled)
 417                 return;
 418
 419         /* Arm the timer for 1-2 ticks later if not already. */
 420         if (!timer_pending(&pcpu->cpu_timer)) {
 421                 pcpu->timer_idlecancel = 0;
 422                 cpufreq_interactive_timer_resched(pcpu);
 423         } else if (!governidle &&
 424                    time_after_eq(jiffies, pcpu->cpu_timer.expires)) {
 425                 del_timer(&pcpu->cpu_timer);
 426                 cpufreq_interactive_timer(smp_processor_id());
 427         }
 428 }
 429
 430 static int cpufreq_interactive_speedchange_task(void *data)
 431 {
 432         unsigned int cpu;
 433         cpumask_t tmp_mask;
 434         unsigned long flags;
 435         struct cpufreq_interactive_cpuinfo *pcpu;
 436
 437         while (1) {
 438                 set_current_state(TASK_INTERRUPTIBLE);
 439                 spin_lock_irqsave(&speedchange_cpumask_lock, flags);
 440
 441                 if (cpumask_empty(&speedchange_cpumask)) {
 442                         spin_unlock_irqrestore(&speedchange_cpumask_lock,
 443                                                flags);
 444                         schedule();
 445
 446                         if (kthread_should_stop())
 447                                 break;
 448
 449                         spin_lock_irqsave(&speedchange_cpumask_lock, flags);
 450                 }
 451
 452                 set_current_state(TASK_RUNNING);
 453                 tmp_mask = speedchange_cpumask;
 454                 cpumask_clear(&speedchange_cpumask);
 455                 spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);
 456
 457                 for_each_cpu(cpu, &tmp_mask) {
 458                         unsigned int j;
 459                         unsigned int max_freq = 0;
 460
 461                         pcpu = &per_cpu(cpuinfo, cpu);
 462                         smp_rmb();
 463
 464                         if (!pcpu->governor_enabled)
 465                                 continue;
 466
 467                         for_each_cpu(j, pcpu->policy->cpus) {
 468                                 struct cpufreq_interactive_cpuinfo *pjcpu =
 469                                         &per_cpu(cpuinfo, j);
 470
 471                                 if (pjcpu->target_freq > max_freq)
 472                                         max_freq = pjcpu->target_freq;
 473                         }
 474
 475                         if (max_freq != pcpu->policy->cur)
 476                                 __cpufreq_driver_target(pcpu->policy,
 477                                                         max_freq,
 478                                                         CPUFREQ_RELATION_H);
 479                         trace_cpufreq_interactive_setspeed(cpu,
 480                                                      pcpu->target_freq,
 481                                                      pcpu->policy->cur);
 482                 }
 483         }
 484
 485         return 0;
 486 }
 487
 488 static void cpufreq_interactive_boost(void)
 489 {
 490         int i;
 491         int anyboost = 0;
 492         unsigned long flags;
 493         struct cpufreq_interactive_cpuinfo *pcpu;
 494
 495         spin_lock_irqsave(&speedchange_cpumask_lock, flags);
 496
 497         for_each_online_cpu(i) {
 498                 pcpu = &per_cpu(cpuinfo, i);
 499
 500                 if (pcpu->target_freq < hispeed_freq) {
 501                         pcpu->target_freq = hispeed_freq;
 502                         cpumask_set_cpu(i, &speedchange_cpumask);
 503                         pcpu->target_set_time_in_idle =
 504                                 get_cpu_idle_time_us(i, &pcpu->target_set_time);
 505                         pcpu->hispeed_validate_time = pcpu->target_set_time;
 506                         anyboost = 1;
 507                 }
 508
 509                 /*
 510                  * Set floor freq and (re)start timer for when last
 511                  * validated.
 512                  */
 513
 514                 pcpu->floor_freq = hispeed_freq;
 515                 pcpu->floor_validate_time = ktime_to_us(ktime_get());
 516         }
 517
 518         spin_unlock_irqrestore(&speedchange_cpumask_lock, flags);
 519
 520         if (anyboost)
 521                 wake_up_process(speedchange_task);
 522 }
 523
 524 static ssize_t show_target_loads(
 525         struct kobject *kobj, struct attribute *attr, char *buf)
 526 {
 527         int i;
 528         ssize_t ret = 0;
 529
 530         spin_lock(&target_loads_lock);
 531
 532         for (i = 0; i < ntarget_loads; i++)
 533                 ret += sprintf(buf + ret, "%u%s", target_loads[i],
 534                                i & 0x1 ? ":" : " ");
 535
 536         ret += sprintf(buf + ret, "\n");
 537         spin_unlock(&target_loads_lock);
 538         return ret;
 539 }
 540
 541 static ssize_t store_target_loads(
 542         struct kobject *kobj, struct attribute *attr, const char *buf,
 543         size_t count)
 544 {
 545         int ret;
 546         const char *cp;
 547         unsigned int *new_target_loads = NULL;
 548         int ntokens = 1;
 549         int i;
 550
 551         cp = buf;
 552         while ((cp = strpbrk(cp + 1, " :")))
 553                 ntokens++;
 554
 555         if (!(ntokens & 0x1))
 556                 goto err_inval;
 557
 558         new_target_loads = kmalloc(ntokens * sizeof(unsigned int), GFP_KERNEL);
 559         if (!new_target_loads) {
 560                 ret = -ENOMEM;
 561                 goto err;
 562         }
 563
 564         cp = buf;
 565         i = 0;
 566         while (i < ntokens) {
 567                 if (sscanf(cp, "%u", &new_target_loads[i++]) != 1)
 568                         goto err_inval;
 569
 570                 cp = strpbrk(cp, " :");
 571                 if (!cp)
 572                         break;
 573                 cp++;
 574         }
 575
 576         if (i != ntokens)
 577                 goto err_inval;
 578
 579         spin_lock(&target_loads_lock);
 580         if (target_loads != default_target_loads)
 581                 kfree(target_loads);
 582         target_loads = new_target_loads;
 583         ntarget_loads = ntokens;
 584         spin_unlock(&target_loads_lock);
 585         return count;
 586
 587 err_inval:
 588         ret = -EINVAL;
 589 err:
 590         kfree(new_target_loads);
 591         return ret;
 592 }
 593
 594 static struct global_attr target_loads_attr =
 595         __ATTR(target_loads, S_IRUGO | S_IWUSR,
 596                 show_target_loads, store_target_loads);
 597
 598 static ssize_t show_hispeed_freq(struct kobject *kobj,
 599                                  struct attribute *attr, char *buf)
 600 {
 601         return sprintf(buf, "%u\n", hispeed_freq);
 602 }
 603
 604 static ssize_t store_hispeed_freq(struct kobject *kobj,
 605                                   struct attribute *attr, const char *buf,
 606                                   size_t count)
 607 {
 608         int ret;
 609         long unsigned int val;
 610
 611         ret = strict_strtoul(buf, 0, &val);
 612         if (ret < 0)
 613                 return ret;
 614         hispeed_freq = val;
 615         return count;
 616 }
 617
 618 static struct global_attr hispeed_freq_attr = __ATTR(hispeed_freq, 0644,
 619                 show_hispeed_freq, store_hispeed_freq);
 620
 621
 622 static ssize_t show_go_hispeed_load(struct kobject *kobj,
 623                                      struct attribute *attr, char *buf)
 624 {
 625         return sprintf(buf, "%lu\n", go_hispeed_load);
 626 }
 627
 628 static ssize_t store_go_hispeed_load(struct kobject *kobj,
 629                         struct attribute *attr, const char *buf, size_t count)
 630 {
 631         int ret;
 632         unsigned long val;
 633
 634         ret = strict_strtoul(buf, 0, &val);
 635         if (ret < 0)
 636                 return ret;
 637         go_hispeed_load = val;
 638         return count;
 639 }
 640
 641 static struct global_attr go_hispeed_load_attr = __ATTR(go_hispeed_load, 0644,
 642                 show_go_hispeed_load, store_go_hispeed_load);
 643
 644 static ssize_t show_min_sample_time(struct kobject *kobj,
 645                                 struct attribute *attr, char *buf)
 646 {
 647         return sprintf(buf, "%lu\n", min_sample_time);
 648 }
 649
 650 static ssize_t store_min_sample_time(struct kobject *kobj,
 651                         struct attribute *attr, const char *buf, size_t count)
 652 {
 653         int ret;
 654         unsigned long val;
 655
 656         ret = strict_strtoul(buf, 0, &val);
 657         if (ret < 0)
 658                 return ret;
 659         min_sample_time = val;
 660         return count;
 661 }
 662
 663 static struct global_attr min_sample_time_attr = __ATTR(min_sample_time, 0644,
 664                 show_min_sample_time, store_min_sample_time);
 665
 666 static ssize_t show_above_hispeed_delay(struct kobject *kobj,
 667                                         struct attribute *attr, char *buf)
 668 {
 669         return sprintf(buf, "%lu\n", above_hispeed_delay_val);
 670 }
 671
 672 static ssize_t store_above_hispeed_delay(struct kobject *kobj,
 673                                          struct attribute *attr,
 674                                          const char *buf, size_t count)
 675 {
 676         int ret;
 677         unsigned long val;
 678
 679         ret = strict_strtoul(buf, 0, &val);
 680         if (ret < 0)
 681                 return ret;
 682         above_hispeed_delay_val = val;
 683         return count;
 684 }
 685
 686 define_one_global_rw(above_hispeed_delay);
 687
 688 static ssize_t show_timer_rate(struct kobject *kobj,
 689                         struct attribute *attr, char *buf)
 690 {
 691         return sprintf(buf, "%lu\n", timer_rate);
 692 }
 693
 694 static ssize_t store_timer_rate(struct kobject *kobj,
 695                         struct attribute *attr, const char *buf, size_t count)
 696 {
 697         int ret;
 698         unsigned long val;
 699
 700         ret = strict_strtoul(buf, 0, &val);
 701         if (ret < 0)
 702                 return ret;
 703         timer_rate = val;
 704         return count;
 705 }
 706
 707 static struct global_attr timer_rate_attr = __ATTR(timer_rate, 0644,
 708                 show_timer_rate, store_timer_rate);
 709
 710 static ssize_t show_boost(struct kobject *kobj, struct attribute *attr,
 711                           char *buf)
 712 {
 713         return sprintf(buf, "%d\n", boost_val);
 714 }
 715
 716 static ssize_t store_boost(struct kobject *kobj, struct attribute *attr,
 717                            const char *buf, size_t count)
 718 {
 719         int ret;
 720         unsigned long val;
 721
 722         ret = kstrtoul(buf, 0, &val);
 723         if (ret < 0)
 724                 return ret;
 725
 726         boost_val = val;
 727
 728         if (boost_val) {
 729                 trace_cpufreq_interactive_boost("on");
 730                 cpufreq_interactive_boost();
 731         } else {
 732                 trace_cpufreq_interactive_unboost("off");
 733         }
 734
 735         return count;
 736 }
 737
 738 define_one_global_rw(boost);
 739
 740 static ssize_t store_boostpulse(struct kobject *kobj, struct attribute *attr,
 741                                 const char *buf, size_t count)
 742 {
 743         int ret;
 744         unsigned long val;
 745
 746         ret = kstrtoul(buf, 0, &val);
 747         if (ret < 0)
 748                 return ret;
 749
 750         trace_cpufreq_interactive_boost("pulse");
 751         cpufreq_interactive_boost();
 752         return count;
 753 }
 754
 755 static struct global_attr boostpulse =
 756         __ATTR(boostpulse, 0200, NULL, store_boostpulse);
 757
 758 static struct attribute *interactive_attributes[] = {
 759         &target_loads_attr.attr,
 760         &hispeed_freq_attr.attr,
 761         &go_hispeed_load_attr.attr,
 762         &above_hispeed_delay.attr,
 763         &min_sample_time_attr.attr,
 764         &timer_rate_attr.attr,
 765         &boost.attr,
 766         &boostpulse.attr,
 767         NULL,
 768 };
 769
 770 static struct attribute_group interactive_attr_group = {
 771         .attrs = interactive_attributes,
 772         .name = "interactive",
 773 };
 774
 775 static int cpufreq_interactive_idle_notifier(struct notifier_block *nb,
 776                                              unsigned long val,
 777                                              void *data)
 778 {
 779         switch (val) {
 780         case IDLE_START:
 781                 cpufreq_interactive_idle_start();
 782                 break;
 783         case IDLE_END:
 784                 cpufreq_interactive_idle_end();
 785                 break;
 786         }
 787
 788         return 0;
 789 }
 790
 791 static struct notifier_block cpufreq_interactive_idle_nb = {
 792         .notifier_call = cpufreq_interactive_idle_notifier,
 793 };
 794
 795 static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
 796                 unsigned int event)
 797 {
 798         int rc;
 799         unsigned int j;
 800         struct cpufreq_interactive_cpuinfo *pcpu;
 801         struct cpufreq_frequency_table *freq_table;
 802
 803         switch (event) {
 804         case CPUFREQ_GOV_START:
 805                 if (!cpu_online(policy->cpu))
 806                         return -EINVAL;
 807
 808                 freq_table =
 809                         cpufreq_frequency_get_table(policy->cpu);
 810                 if (!hispeed_freq)
 811                         hispeed_freq = policy->max;
 812
 813                 for_each_cpu(j, policy->cpus) {
 814                         pcpu = &per_cpu(cpuinfo, j);
 815                         pcpu->policy = policy;
 816                         pcpu->target_freq = policy->cur;
 817                         pcpu->freq_table = freq_table;
 818                         pcpu->target_set_time_in_idle =
 819                                 get_cpu_idle_time_us(j,
 820                                              &pcpu->target_set_time);
 821                         pcpu->floor_freq = pcpu->target_freq;
 822                         pcpu->floor_validate_time =
 823                                 pcpu->target_set_time;
 824                         pcpu->hispeed_validate_time =
 825                                 pcpu->target_set_time;
 826                         pcpu->governor_enabled = 1;
 827                         smp_wmb();
 828                         pcpu->cpu_timer.expires =
 829                                 jiffies + usecs_to_jiffies(timer_rate);
 830                         add_timer_on(&pcpu->cpu_timer, j);
 831                 }
 832
 833                 /*
 834                  * Do not register the idle hook and create sysfs
 835                  * entries if we have already done so.
 836                  */
 837                 if (atomic_inc_return(&active_count) > 1)
 838                         return 0;
 839
 840                 rc = sysfs_create_group(cpufreq_global_kobject,
 841                                 &interactive_attr_group);
 842                 if (rc)
 843                         return rc;
 844
 845                 idle_notifier_register(&cpufreq_interactive_idle_nb);
 846                 break;
 847
 848         case CPUFREQ_GOV_STOP:
 849                 for_each_cpu(j, policy->cpus) {
 850                         pcpu = &per_cpu(cpuinfo, j);
 851                         pcpu->governor_enabled = 0;
 852                         smp_wmb();
 853                         del_timer_sync(&pcpu->cpu_timer);
 854                 }
 855
 856                 if (atomic_dec_return(&active_count) > 0)
 857                         return 0;
 858
 859                 idle_notifier_unregister(&cpufreq_interactive_idle_nb);
 860                 sysfs_remove_group(cpufreq_global_kobject,
 861                                 &interactive_attr_group);
 862
 863                 break;
 864
 865         case CPUFREQ_GOV_LIMITS:
 866                 if (policy->max < policy->cur)
 867                         __cpufreq_driver_target(policy,
 868                                         policy->max, CPUFREQ_RELATION_H);
 869                 else if (policy->min > policy->cur)
 870                         __cpufreq_driver_target(policy,
 871                                         policy->min, CPUFREQ_RELATION_L);
 872                 break;
 873         }
 874         return 0;
 875 }
 876
 877 static int __init cpufreq_interactive_init(void)
 878 {
 879         unsigned int i;
 880         struct cpufreq_interactive_cpuinfo *pcpu;
 881         struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
 882
 883         go_hispeed_load = DEFAULT_GO_HISPEED_LOAD;
 884         min_sample_time = DEFAULT_MIN_SAMPLE_TIME;
 885         above_hispeed_delay_val = DEFAULT_ABOVE_HISPEED_DELAY;
 886         timer_rate = DEFAULT_TIMER_RATE;
 887
 888         /* Initalize per-cpu timers */
 889         for_each_possible_cpu(i) {
 890                 pcpu = &per_cpu(cpuinfo, i);
 891                 if (governidle)
 892                         init_timer(&pcpu->cpu_timer);
 893                 else
 894                         init_timer_deferrable(&pcpu->cpu_timer);
 895                 pcpu->cpu_timer.function = cpufreq_interactive_timer;
 896                 pcpu->cpu_timer.data = i;
 897         }
 898
 899         spin_lock_init(&target_loads_lock);
 900         spin_lock_init(&speedchange_cpumask_lock);
 901         speedchange_task =
 902                 kthread_create(cpufreq_interactive_speedchange_task, NULL,
 903                                "cfinteractive");
 904         if (IS_ERR(speedchange_task))
 905                 return PTR_ERR(speedchange_task);
 906
 907         sched_setscheduler_nocheck(speedchange_task, SCHED_FIFO, &param);
 908         get_task_struct(speedchange_task);
 909
 910         /* NB: wake up so the thread does not look hung to the freezer */
 911         wake_up_process(speedchange_task);
 912
 913         return cpufreq_register_governor(&cpufreq_gov_interactive);
 914 }
 915
 916 #ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE
 917 fs_initcall(cpufreq_interactive_init);
 918 #else
 919 module_init(cpufreq_interactive_init);
 920 #endif
 921
 922 static void __exit cpufreq_interactive_exit(void)
 923 {
 924         cpufreq_unregister_governor(&cpufreq_gov_interactive);
 925         kthread_stop(speedchange_task);
 926         put_task_struct(speedchange_task);
 927 }
 928
 929 module_exit(cpufreq_interactive_exit);
 930
 931 MODULE_AUTHOR("Mike Chan <mike@android.com>");
 932 MODULE_DESCRIPTION("'cpufreq_interactive' - A cpufreq governor for "
 933         "Latency sensitive workloads");
 934 MODULE_LICENSE("GPL");