drivers/cpufreq/cpufreq_interactive.c

   1 /*
   2  * drivers/cpufreq/cpufreq_interactive.c
   3  *
   4  * Copyright (C) 2010 Google, Inc.
   5  *
   6  * This software is licensed under the terms of the GNU General Public
   7  * License version 2, as published by the Free Software Foundation, and
   8  * may be copied, distributed, and modified under those terms.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * Author: Mike Chan (mike@android.com)
  16  *
  17  */
  18
  19 #include <linux/cpu.h>
  20 #include <linux/cpumask.h>
  21 #include <linux/cpufreq.h>
  22 #include <linux/module.h>
  23 #include <linux/mutex.h>
  24 #include <linux/sched.h>
  25 #include <linux/sched/rt.h>
  26 #include <linux/tick.h>
  27 #include <linux/time.h>
  28 #include <linux/timer.h>
  29 #include <linux/workqueue.h>
  30 #include <linux/kthread.h>
  31 #include <linux/mutex.h>
  32 #include <linux/slab.h>
  33 #include <linux/input.h>
  34
  35 #define CREATE_TRACE_POINTS
  36 #include <trace/events/cpufreq_interactive.h>
  37
  38 #include <asm/cputime.h>
  39
  40 static atomic_t active_count = ATOMIC_INIT(0);
  41
  42 struct cpufreq_interactive_cpuinfo {
  43         struct timer_list cpu_timer;
  44         int timer_idlecancel;
  45         u64 time_in_idle;
  46         u64 idle_exit_time;
  47         u64 timer_run_time;
  48         int idling;
  49         u64 target_set_time;
  50         u64 target_set_time_in_idle;
  51         u64 target_validate_time;
  52         struct cpufreq_policy *policy;
  53         struct cpufreq_frequency_table *freq_table;
  54         unsigned int target_freq;
  55         int governor_enabled;
  56 };
  57
  58 static DEFINE_PER_CPU(struct cpufreq_interactive_cpuinfo, cpuinfo);
  59
  60 /* Workqueues handle frequency scaling */
  61 static struct task_struct *up_task;
  62 static struct workqueue_struct *down_wq;
  63 static struct work_struct freq_scale_down_work;
  64 static cpumask_t up_cpumask;
  65 static spinlock_t up_cpumask_lock;
  66 static cpumask_t down_cpumask;
  67 static spinlock_t down_cpumask_lock;
  68 static struct mutex set_speed_lock;
  69
  70 /* Hi speed to bump to from lo speed when load burst (default max) */
  71 static u64 hispeed_freq;
  72
  73 /* Go to hi speed when CPU load at or above this value. */
  74 #define DEFAULT_GO_HISPEED_LOAD 85
  75 static unsigned long go_hispeed_load;
  76
  77 /*
  78  * The minimum amount of time to spend at a frequency before we can ramp down.
  79  */
  80 #define DEFAULT_MIN_SAMPLE_TIME (80 * USEC_PER_MSEC)
  81 static unsigned long min_sample_time;
  82
  83 /*
  84  * The sample rate of the timer used to increase frequency
  85  */
  86 #define DEFAULT_TIMER_RATE (20 * USEC_PER_MSEC)
  87 static unsigned long timer_rate;
  88
  89 /*
  90  * Wait this long before raising speed above hispeed, by default a single
  91  * timer interval.
  92  */
  93 #define DEFAULT_ABOVE_HISPEED_DELAY DEFAULT_TIMER_RATE
  94 static unsigned long above_hispeed_delay_val;
  95
  96 /*
  97  * Boost to hispeed on touchscreen input.
  98  */
  99
 100 static int input_boost_val;
 101
 102 struct cpufreq_interactive_inputopen {
 103         struct input_handle *handle;
 104         struct work_struct inputopen_work;
 105 };
 106
 107 static struct cpufreq_interactive_inputopen inputopen;
 108
 109 static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
 110                 unsigned int event);
 111
 112 #ifndef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE
 113 static
 114 #endif
 115 struct cpufreq_governor cpufreq_gov_interactive = {
 116         .name = "interactive",
 117         .governor = cpufreq_governor_interactive,
 118         .max_transition_latency = 10000000,
 119         .owner = THIS_MODULE,
 120 };
 121
 122 static void cpufreq_interactive_timer(unsigned long data)
 123 {
 124         unsigned int delta_idle;
 125         unsigned int delta_time;
 126         int cpu_load;
 127         int load_since_change;
 128         u64 time_in_idle;
 129         u64 idle_exit_time;
 130         struct cpufreq_interactive_cpuinfo *pcpu =
 131                 &per_cpu(cpuinfo, data);
 132         u64 now_idle;
 133         unsigned int new_freq;
 134         unsigned int index;
 135         unsigned long flags;
 136
 137         smp_rmb();
 138
 139         if (!pcpu->governor_enabled)
 140                 goto exit;
 141
 142         /*
 143          * Once pcpu->timer_run_time is updated to >= pcpu->idle_exit_time,
 144          * this lets idle exit know the current idle time sample has
 145          * been processed, and idle exit can generate a new sample and
 146          * re-arm the timer.  This prevents a concurrent idle
 147          * exit on that CPU from writing a new set of info at the same time
 148          * the timer function runs (the timer function can't use that info
 149          * until more time passes).
 150          */
 151         time_in_idle = pcpu->time_in_idle;
 152         idle_exit_time = pcpu->idle_exit_time;
 153         now_idle = get_cpu_idle_time_us(data, &pcpu->timer_run_time);
 154         smp_wmb();
 155
 156         /* If we raced with cancelling a timer, skip. */
 157         if (!idle_exit_time)
 158                 goto exit;
 159
 160         delta_idle = (unsigned int)(now_idle - time_in_idle);
 161         delta_time = (unsigned int)(pcpu->timer_run_time - idle_exit_time);
 162
 163         /*
 164          * If timer ran less than 1ms after short-term sample started, retry.
 165          */
 166         if (delta_time < 1000)
 167                 goto rearm;
 168
 169         if (delta_idle > delta_time)
 170                 cpu_load = 0;
 171         else
 172                 cpu_load = 100 * (delta_time - delta_idle) / delta_time;
 173
 174         delta_idle = (unsigned int)(now_idle - pcpu->target_set_time_in_idle);
 175         delta_time = (unsigned int)(pcpu->timer_run_time -
 176                                     pcpu->target_set_time);
 177
 178         if ((delta_time == 0) || (delta_idle > delta_time))
 179                 load_since_change = 0;
 180         else
 181                 load_since_change =
 182                         100 * (delta_time - delta_idle) / delta_time;
 183
 184         /*
 185          * Choose greater of short-term load (since last idle timer
 186          * started or timer function re-armed itself) or long-term load
 187          * (since last frequency change).
 188          */
 189         if (load_since_change > cpu_load)
 190                 cpu_load = load_since_change;
 191
 192         if (cpu_load >= go_hispeed_load) {
 193                 if (pcpu->target_freq <= pcpu->policy->min) {
 194                         new_freq = hispeed_freq;
 195                 } else {
 196                         new_freq = pcpu->policy->max * cpu_load / 100;
 197
 198                         if (new_freq < hispeed_freq)
 199                                 new_freq = hispeed_freq;
 200
 201                         if (pcpu->target_freq == hispeed_freq &&
 202                             new_freq > hispeed_freq &&
 203                             pcpu->timer_run_time - pcpu->target_set_time
 204                             < above_hispeed_delay_val) {
 205                                 trace_cpufreq_interactive_notyet(data, cpu_load,
 206                                                                  pcpu->target_freq,
 207                                                                  new_freq);
 208                                 goto rearm;
 209                         }
 210                 }
 211         } else {
 212                 new_freq = pcpu->policy->max * cpu_load / 100;
 213         }
 214
 215         if (cpufreq_frequency_table_target(pcpu->policy, pcpu->freq_table,
 216                                            new_freq, CPUFREQ_RELATION_H,
 217                                            &index)) {
 218                 pr_warn_once("timer %d: cpufreq_frequency_table_target error\n",
 219                              (int) data);
 220                 goto rearm;
 221         }
 222
 223         new_freq = pcpu->freq_table[index].frequency;
 224
 225         /*
 226          * Do not scale down unless we have been at this frequency for the
 227          * minimum sample time since last validated.
 228          */
 229         if (new_freq < pcpu->target_freq) {
 230                 if (pcpu->timer_run_time - pcpu->target_validate_time
 231                     < min_sample_time) {
 232                         trace_cpufreq_interactive_notyet(data, cpu_load,
 233                                          pcpu->target_freq, new_freq);
 234                         goto rearm;
 235                 }
 236         }
 237
 238         pcpu->target_validate_time = pcpu->timer_run_time;
 239
 240         if (pcpu->target_freq == new_freq) {
 241                 trace_cpufreq_interactive_already(data, cpu_load,
 242                                                   pcpu->target_freq, new_freq);
 243                 goto rearm_if_notmax;
 244         }
 245
 246         trace_cpufreq_interactive_target(data, cpu_load, pcpu->target_freq,
 247                                          new_freq);
 248         pcpu->target_set_time_in_idle = now_idle;
 249         pcpu->target_set_time = pcpu->timer_run_time;
 250
 251         if (new_freq < pcpu->target_freq) {
 252                 pcpu->target_freq = new_freq;
 253                 spin_lock_irqsave(&down_cpumask_lock, flags);
 254                 cpumask_set_cpu(data, &down_cpumask);
 255                 spin_unlock_irqrestore(&down_cpumask_lock, flags);
 256                 queue_work(down_wq, &freq_scale_down_work);
 257         } else {
 258                 pcpu->target_freq = new_freq;
 259                 spin_lock_irqsave(&up_cpumask_lock, flags);
 260                 cpumask_set_cpu(data, &up_cpumask);
 261                 spin_unlock_irqrestore(&up_cpumask_lock, flags);
 262                 wake_up_process(up_task);
 263         }
 264
 265 rearm_if_notmax:
 266         /*
 267          * Already set max speed and don't see a need to change that,
 268          * wait until next idle to re-evaluate, don't need timer.
 269          */
 270         if (pcpu->target_freq == pcpu->policy->max)
 271                 goto exit;
 272
 273 rearm:
 274         if (!timer_pending(&pcpu->cpu_timer)) {
 275                 /*
 276                  * If already at min: if that CPU is idle, don't set timer.
 277                  * Else cancel the timer if that CPU goes idle.  We don't
 278                  * need to re-evaluate speed until the next idle exit.
 279                  */
 280                 if (pcpu->target_freq == pcpu->policy->min) {
 281                         smp_rmb();
 282
 283                         if (pcpu->idling)
 284                                 goto exit;
 285
 286                         pcpu->timer_idlecancel = 1;
 287                 }
 288
 289                 pcpu->time_in_idle = get_cpu_idle_time_us(
 290                         data, &pcpu->idle_exit_time);
 291                 mod_timer(&pcpu->cpu_timer,
 292                           jiffies + usecs_to_jiffies(timer_rate));
 293         }
 294
 295 exit:
 296         return;
 297 }
 298
 299 static void cpufreq_interactive_idle_start(void)
 300 {
 301         struct cpufreq_interactive_cpuinfo *pcpu =
 302                 &per_cpu(cpuinfo, smp_processor_id());
 303         int pending;
 304
 305         if (!pcpu->governor_enabled)
 306                 return;
 307
 308         pcpu->idling = 1;
 309         smp_wmb();
 310         pending = timer_pending(&pcpu->cpu_timer);
 311
 312         if (pcpu->target_freq != pcpu->policy->min) {
 313 #ifdef CONFIG_SMP
 314                 /*
 315                  * Entering idle while not at lowest speed.  On some
 316                  * platforms this can hold the other CPU(s) at that speed
 317                  * even though the CPU is idle. Set a timer to re-evaluate
 318                  * speed so this idle CPU doesn't hold the other CPUs above
 319                  * min indefinitely.  This should probably be a quirk of
 320                  * the CPUFreq driver.
 321                  */
 322                 if (!pending) {
 323                         pcpu->time_in_idle = get_cpu_idle_time_us(
 324                                 smp_processor_id(), &pcpu->idle_exit_time);
 325                         pcpu->timer_idlecancel = 0;
 326                         mod_timer(&pcpu->cpu_timer,
 327                                   jiffies + usecs_to_jiffies(timer_rate));
 328                 }
 329 #endif
 330         } else {
 331                 /*
 332                  * If at min speed and entering idle after load has
 333                  * already been evaluated, and a timer has been set just in
 334                  * case the CPU suddenly goes busy, cancel that timer.  The
 335                  * CPU didn't go busy; we'll recheck things upon idle exit.
 336                  */
 337                 if (pending && pcpu->timer_idlecancel) {
 338                         del_timer(&pcpu->cpu_timer);
 339                         /*
 340                          * Ensure last timer run time is after current idle
 341                          * sample start time, so next idle exit will always
 342                          * start a new idle sampling period.
 343                          */
 344                         pcpu->idle_exit_time = 0;
 345                         pcpu->timer_idlecancel = 0;
 346                 }
 347         }
 348
 349 }
 350
 351 static void cpufreq_interactive_idle_end(void)
 352 {
 353         struct cpufreq_interactive_cpuinfo *pcpu =
 354                 &per_cpu(cpuinfo, smp_processor_id());
 355
 356         pcpu->idling = 0;
 357         smp_wmb();
 358
 359         /*
 360          * Arm the timer for 1-2 ticks later if not already, and if the timer
 361          * function has already processed the previous load sampling
 362          * interval.  (If the timer is not pending but has not processed
 363          * the previous interval, it is probably racing with us on another
 364          * CPU.  Let it compute load based on the previous sample and then
 365          * re-arm the timer for another interval when it's done, rather
 366          * than updating the interval start time to be "now", which doesn't
 367          * give the timer function enough time to make a decision on this
 368          * run.)
 369          */
 370         if (timer_pending(&pcpu->cpu_timer) == 0 &&
 371             pcpu->timer_run_time >= pcpu->idle_exit_time &&
 372             pcpu->governor_enabled) {
 373                 pcpu->time_in_idle =
 374                         get_cpu_idle_time_us(smp_processor_id(),
 375                                              &pcpu->idle_exit_time);
 376                 pcpu->timer_idlecancel = 0;
 377                 mod_timer(&pcpu->cpu_timer,
 378                           jiffies + usecs_to_jiffies(timer_rate));
 379         }
 380
 381 }
 382
 383 static int cpufreq_interactive_up_task(void *data)
 384 {
 385         unsigned int cpu;
 386         cpumask_t tmp_mask;
 387         unsigned long flags;
 388         struct cpufreq_interactive_cpuinfo *pcpu;
 389
 390         while (1) {
 391                 set_current_state(TASK_INTERRUPTIBLE);
 392                 spin_lock_irqsave(&up_cpumask_lock, flags);
 393
 394                 if (cpumask_empty(&up_cpumask)) {
 395                         spin_unlock_irqrestore(&up_cpumask_lock, flags);
 396                         schedule();
 397
 398                         if (kthread_should_stop())
 399                                 break;
 400
 401                         spin_lock_irqsave(&up_cpumask_lock, flags);
 402                 }
 403
 404                 set_current_state(TASK_RUNNING);
 405                 tmp_mask = up_cpumask;
 406                 cpumask_clear(&up_cpumask);
 407                 spin_unlock_irqrestore(&up_cpumask_lock, flags);
 408
 409                 for_each_cpu(cpu, &tmp_mask) {
 410                         unsigned int j;
 411                         unsigned int max_freq = 0;
 412
 413                         pcpu = &per_cpu(cpuinfo, cpu);
 414                         smp_rmb();
 415
 416                         if (!pcpu->governor_enabled)
 417                                 continue;
 418
 419                         mutex_lock(&set_speed_lock);
 420
 421                         for_each_cpu(j, pcpu->policy->cpus) {
 422                                 struct cpufreq_interactive_cpuinfo *pjcpu =
 423                                         &per_cpu(cpuinfo, j);
 424
 425                                 if (pjcpu->target_freq > max_freq)
 426                                         max_freq = pjcpu->target_freq;
 427                         }
 428
 429                         if (max_freq != pcpu->policy->cur)
 430                                 __cpufreq_driver_target(pcpu->policy,
 431                                                         max_freq,
 432                                                         CPUFREQ_RELATION_H);
 433                         mutex_unlock(&set_speed_lock);
 434                         trace_cpufreq_interactive_up(cpu, pcpu->target_freq,
 435                                                      pcpu->policy->cur);
 436                 }
 437         }
 438
 439         return 0;
 440 }
 441
 442 static void cpufreq_interactive_freq_down(struct work_struct *work)
 443 {
 444         unsigned int cpu;
 445         cpumask_t tmp_mask;
 446         unsigned long flags;
 447         struct cpufreq_interactive_cpuinfo *pcpu;
 448
 449         spin_lock_irqsave(&down_cpumask_lock, flags);
 450         tmp_mask = down_cpumask;
 451         cpumask_clear(&down_cpumask);
 452         spin_unlock_irqrestore(&down_cpumask_lock, flags);
 453
 454         for_each_cpu(cpu, &tmp_mask) {
 455                 unsigned int j;
 456                 unsigned int max_freq = 0;
 457
 458                 pcpu = &per_cpu(cpuinfo, cpu);
 459                 smp_rmb();
 460
 461                 if (!pcpu->governor_enabled)
 462                         continue;
 463
 464                 mutex_lock(&set_speed_lock);
 465
 466                 for_each_cpu(j, pcpu->policy->cpus) {
 467                         struct cpufreq_interactive_cpuinfo *pjcpu =
 468                                 &per_cpu(cpuinfo, j);
 469
 470                         if (pjcpu->target_freq > max_freq)
 471                                 max_freq = pjcpu->target_freq;
 472                 }
 473
 474                 if (max_freq != pcpu->policy->cur)
 475                         __cpufreq_driver_target(pcpu->policy, max_freq,
 476                                                 CPUFREQ_RELATION_H);
 477
 478                 mutex_unlock(&set_speed_lock);
 479                 trace_cpufreq_interactive_down(cpu, pcpu->target_freq,
 480                                                pcpu->policy->cur);
 481         }
 482 }
 483
 484 static void cpufreq_interactive_boost(void)
 485 {
 486         int i;
 487         int anyboost = 0;
 488         unsigned long flags;
 489         struct cpufreq_interactive_cpuinfo *pcpu;
 490
 491         trace_cpufreq_interactive_boost(hispeed_freq);
 492         spin_lock_irqsave(&up_cpumask_lock, flags);
 493
 494         for_each_online_cpu(i) {
 495                 pcpu = &per_cpu(cpuinfo, i);
 496
 497                 if (pcpu->target_freq < hispeed_freq) {
 498                         pcpu->target_freq = hispeed_freq;
 499                         cpumask_set_cpu(i, &up_cpumask);
 500                         pcpu->target_set_time_in_idle =
 501                                 get_cpu_idle_time_us(i, &pcpu->target_set_time);
 502                         anyboost = 1;
 503                 }
 504
 505                 /*
 506                  * Refresh time at which current (possibly being
 507                  * boosted) speed last validated (reset timer for
 508                  * allowing speed to drop).
 509                  */
 510
 511                 pcpu->target_validate_time = ktime_to_us(ktime_get());
 512         }
 513
 514         spin_unlock_irqrestore(&up_cpumask_lock, flags);
 515
 516         if (anyboost)
 517                 wake_up_process(up_task);
 518 }
 519
 520 static void cpufreq_interactive_input_event(struct input_handle *handle,
 521                                             unsigned int type,
 522                                             unsigned int code, int value)
 523 {
 524         if (input_boost_val && type == EV_SYN && code == SYN_REPORT)
 525                 cpufreq_interactive_boost();
 526 }
 527
 528 static void cpufreq_interactive_input_open(struct work_struct *w)
 529 {
 530         struct cpufreq_interactive_inputopen *io =
 531                 container_of(w, struct cpufreq_interactive_inputopen,
 532                              inputopen_work);
 533         int error;
 534
 535         error = input_open_device(io->handle);
 536         if (error)
 537                 input_unregister_handle(io->handle);
 538 }
 539
 540 static int cpufreq_interactive_input_connect(struct input_handler *handler,
 541                                              struct input_dev *dev,
 542                                              const struct input_device_id *id)
 543 {
 544         struct input_handle *handle;
 545         int error;
 546
 547         pr_info("%s: connect to %s\n", __func__, dev->name);
 548         handle = kzalloc(sizeof(struct input_handle), GFP_KERNEL);
 549         if (!handle)
 550                 return -ENOMEM;
 551
 552         handle->dev = dev;
 553         handle->handler = handler;
 554         handle->name = "cpufreq_interactive";
 555
 556         error = input_register_handle(handle);
 557         if (error)
 558                 goto err;
 559
 560         inputopen.handle = handle;
 561         queue_work(down_wq, &inputopen.inputopen_work);
 562         return 0;
 563 err:
 564         kfree(handle);
 565         return error;
 566 }
 567
 568 static void cpufreq_interactive_input_disconnect(struct input_handle *handle)
 569 {
 570         input_close_device(handle);
 571         input_unregister_handle(handle);
 572         kfree(handle);
 573 }
 574
 575 static const struct input_device_id cpufreq_interactive_ids[] = {
 576         {
 577                 .flags = INPUT_DEVICE_ID_MATCH_EVBIT |
 578                          INPUT_DEVICE_ID_MATCH_ABSBIT,
 579                 .evbit = { BIT_MASK(EV_ABS) },
 580                 .absbit = { [BIT_WORD(ABS_MT_POSITION_X)] =
 581                             BIT_MASK(ABS_MT_POSITION_X) |
 582                             BIT_MASK(ABS_MT_POSITION_Y) },
 583         }, /* multi-touch touchscreen */
 584         {
 585                 .flags = INPUT_DEVICE_ID_MATCH_KEYBIT |
 586                          INPUT_DEVICE_ID_MATCH_ABSBIT,
 587                 .keybit = { [BIT_WORD(BTN_TOUCH)] = BIT_MASK(BTN_TOUCH) },
 588                 .absbit = { [BIT_WORD(ABS_X)] =
 589                             BIT_MASK(ABS_X) | BIT_MASK(ABS_Y) },
 590         }, /* touchpad */
 591         { },
 592 };
 593
 594 static struct input_handler cpufreq_interactive_input_handler = {
 595         .event          = cpufreq_interactive_input_event,
 596         .connect        = cpufreq_interactive_input_connect,
 597         .disconnect     = cpufreq_interactive_input_disconnect,
 598         .name           = "cpufreq_interactive",
 599         .id_table       = cpufreq_interactive_ids,
 600 };
 601
 602 static ssize_t show_hispeed_freq(struct kobject *kobj,
 603                                  struct attribute *attr, char *buf)
 604 {
 605         return sprintf(buf, "%llu\n", hispeed_freq);
 606 }
 607
 608 static ssize_t store_hispeed_freq(struct kobject *kobj,
 609                                   struct attribute *attr, const char *buf,
 610                                   size_t count)
 611 {
 612         int ret;
 613         u64 val;
 614
 615         ret = strict_strtoull(buf, 0, &val);
 616         if (ret < 0)
 617                 return ret;
 618         hispeed_freq = val;
 619         return count;
 620 }
 621
 622 static struct global_attr hispeed_freq_attr = __ATTR(hispeed_freq, 0644,
 623                 show_hispeed_freq, store_hispeed_freq);
 624
 625
 626 static ssize_t show_go_hispeed_load(struct kobject *kobj,
 627                                      struct attribute *attr, char *buf)
 628 {
 629         return sprintf(buf, "%lu\n", go_hispeed_load);
 630 }
 631
 632 static ssize_t store_go_hispeed_load(struct kobject *kobj,
 633                         struct attribute *attr, const char *buf, size_t count)
 634 {
 635         int ret;
 636         unsigned long val;
 637
 638         ret = strict_strtoul(buf, 0, &val);
 639         if (ret < 0)
 640                 return ret;
 641         go_hispeed_load = val;
 642         return count;
 643 }
 644
 645 static struct global_attr go_hispeed_load_attr = __ATTR(go_hispeed_load, 0644,
 646                 show_go_hispeed_load, store_go_hispeed_load);
 647
 648 static ssize_t show_min_sample_time(struct kobject *kobj,
 649                                 struct attribute *attr, char *buf)
 650 {
 651         return sprintf(buf, "%lu\n", min_sample_time);
 652 }
 653
 654 static ssize_t store_min_sample_time(struct kobject *kobj,
 655                         struct attribute *attr, const char *buf, size_t count)
 656 {
 657         int ret;
 658         unsigned long val;
 659
 660         ret = strict_strtoul(buf, 0, &val);
 661         if (ret < 0)
 662                 return ret;
 663         min_sample_time = val;
 664         return count;
 665 }
 666
 667 static struct global_attr min_sample_time_attr = __ATTR(min_sample_time, 0644,
 668                 show_min_sample_time, store_min_sample_time);
 669
 670 static ssize_t show_above_hispeed_delay(struct kobject *kobj,
 671                                         struct attribute *attr, char *buf)
 672 {
 673         return sprintf(buf, "%lu\n", above_hispeed_delay_val);
 674 }
 675
 676 static ssize_t store_above_hispeed_delay(struct kobject *kobj,
 677                                          struct attribute *attr,
 678                                          const char *buf, size_t count)
 679 {
 680         int ret;
 681         unsigned long val;
 682
 683         ret = strict_strtoul(buf, 0, &val);
 684         if (ret < 0)
 685                 return ret;
 686         above_hispeed_delay_val = val;
 687         return count;
 688 }
 689
 690 define_one_global_rw(above_hispeed_delay);
 691
 692 static ssize_t show_timer_rate(struct kobject *kobj,
 693                         struct attribute *attr, char *buf)
 694 {
 695         return sprintf(buf, "%lu\n", timer_rate);
 696 }
 697
 698 static ssize_t store_timer_rate(struct kobject *kobj,
 699                         struct attribute *attr, const char *buf, size_t count)
 700 {
 701         int ret;
 702         unsigned long val;
 703
 704         ret = strict_strtoul(buf, 0, &val);
 705         if (ret < 0)
 706                 return ret;
 707         timer_rate = val;
 708         return count;
 709 }
 710
 711 static struct global_attr timer_rate_attr = __ATTR(timer_rate, 0644,
 712                 show_timer_rate, store_timer_rate);
 713
 714 static ssize_t show_input_boost(struct kobject *kobj, struct attribute *attr,
 715                                 char *buf)
 716 {
 717         return sprintf(buf, "%u\n", input_boost_val);
 718 }
 719
 720 static ssize_t store_input_boost(struct kobject *kobj, struct attribute *attr,
 721                                  const char *buf, size_t count)
 722 {
 723         int ret;
 724         unsigned long val;
 725
 726         ret = strict_strtoul(buf, 0, &val);
 727         if (ret < 0)
 728                 return ret;
 729         input_boost_val = val;
 730         return count;
 731 }
 732
 733 define_one_global_rw(input_boost);
 734
 735 static struct attribute *interactive_attributes[] = {
 736         &hispeed_freq_attr.attr,
 737         &go_hispeed_load_attr.attr,
 738         &above_hispeed_delay.attr,
 739         &min_sample_time_attr.attr,
 740         &timer_rate_attr.attr,
 741         &input_boost.attr,
 742         NULL,
 743 };
 744
 745 static struct attribute_group interactive_attr_group = {
 746         .attrs = interactive_attributes,
 747         .name = "interactive",
 748 };
 749
 750 static int cpufreq_governor_interactive(struct cpufreq_policy *policy,
 751                 unsigned int event)
 752 {
 753         int rc;
 754         unsigned int j;
 755         struct cpufreq_interactive_cpuinfo *pcpu;
 756         struct cpufreq_frequency_table *freq_table;
 757
 758         switch (event) {
 759         case CPUFREQ_GOV_START:
 760                 if (!cpu_online(policy->cpu))
 761                         return -EINVAL;
 762
 763                 freq_table =
 764                         cpufreq_frequency_get_table(policy->cpu);
 765
 766                 for_each_cpu(j, policy->cpus) {
 767                         pcpu = &per_cpu(cpuinfo, j);
 768                         pcpu->policy = policy;
 769                         pcpu->target_freq = policy->cur;
 770                         pcpu->freq_table = freq_table;
 771                         pcpu->target_set_time_in_idle =
 772                                 get_cpu_idle_time_us(j,
 773                                              &pcpu->target_set_time);
 774                         pcpu->target_validate_time =
 775                                 pcpu->target_set_time;
 776                         pcpu->governor_enabled = 1;
 777                         smp_wmb();
 778                 }
 779
 780                 if (!hispeed_freq)
 781                         hispeed_freq = policy->max;
 782
 783                 /*
 784                  * Do not register the idle hook and create sysfs
 785                  * entries if we have already done so.
 786                  */
 787                 if (atomic_inc_return(&active_count) > 1)
 788                         return 0;
 789
 790                 rc = sysfs_create_group(cpufreq_global_kobject,
 791                                 &interactive_attr_group);
 792                 if (rc)
 793                         return rc;
 794
 795                 rc = input_register_handler(&cpufreq_interactive_input_handler);
 796                 if (rc)
 797                         pr_warn("%s: failed to register input handler\n",
 798                                 __func__);
 799
 800                 break;
 801
 802         case CPUFREQ_GOV_STOP:
 803                 for_each_cpu(j, policy->cpus) {
 804                         pcpu = &per_cpu(cpuinfo, j);
 805                         pcpu->governor_enabled = 0;
 806                         smp_wmb();
 807                         del_timer_sync(&pcpu->cpu_timer);
 808
 809                         /*
 810                          * Reset idle exit time since we may cancel the timer
 811                          * before it can run after the last idle exit time,
 812                          * to avoid tripping the check in idle exit for a timer
 813                          * that is trying to run.
 814                          */
 815                         pcpu->idle_exit_time = 0;
 816                 }
 817
 818                 flush_work(&freq_scale_down_work);
 819                 if (atomic_dec_return(&active_count) > 0)
 820                         return 0;
 821
 822                 input_unregister_handler(&cpufreq_interactive_input_handler);
 823                 sysfs_remove_group(cpufreq_global_kobject,
 824                                 &interactive_attr_group);
 825
 826                 break;
 827
 828         case CPUFREQ_GOV_LIMITS:
 829                 if (policy->max < policy->cur)
 830                         __cpufreq_driver_target(policy,
 831                                         policy->max, CPUFREQ_RELATION_H);
 832                 else if (policy->min > policy->cur)
 833                         __cpufreq_driver_target(policy,
 834                                         policy->min, CPUFREQ_RELATION_L);
 835                 break;
 836         }
 837         return 0;
 838 }
 839
 840 static int cpufreq_interactive_idle_notifier(struct notifier_block *nb,
 841                                              unsigned long val,
 842                                              void *data)
 843 {
 844         switch (val) {
 845         case IDLE_START:
 846                 cpufreq_interactive_idle_start();
 847                 break;
 848         case IDLE_END:
 849                 cpufreq_interactive_idle_end();
 850                 break;
 851         }
 852
 853         return 0;
 854 }
 855
 856 static struct notifier_block cpufreq_interactive_idle_nb = {
 857         .notifier_call = cpufreq_interactive_idle_notifier,
 858 };
 859
 860 static int __init cpufreq_interactive_init(void)
 861 {
 862         unsigned int i;
 863         struct cpufreq_interactive_cpuinfo *pcpu;
 864         struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
 865
 866         go_hispeed_load = DEFAULT_GO_HISPEED_LOAD;
 867         min_sample_time = DEFAULT_MIN_SAMPLE_TIME;
 868         above_hispeed_delay_val = DEFAULT_ABOVE_HISPEED_DELAY;
 869         timer_rate = DEFAULT_TIMER_RATE;
 870
 871         /* Initalize per-cpu timers */
 872         for_each_possible_cpu(i) {
 873                 pcpu = &per_cpu(cpuinfo, i);
 874                 init_timer(&pcpu->cpu_timer);
 875                 pcpu->cpu_timer.function = cpufreq_interactive_timer;
 876                 pcpu->cpu_timer.data = i;
 877         }
 878
 879         up_task = kthread_create(cpufreq_interactive_up_task, NULL,
 880                                  "kinteractiveup");
 881         if (IS_ERR(up_task))
 882                 return PTR_ERR(up_task);
 883
 884         sched_setscheduler_nocheck(up_task, SCHED_FIFO, &param);
 885         get_task_struct(up_task);
 886
 887         /* No rescuer thread, bind to CPU queuing the work for possibly
 888            warm cache (probably doesn't matter much). */
 889         down_wq = alloc_workqueue("knteractive_down", 0, 1);
 890
 891         if (!down_wq)
 892                 goto err_freeuptask;
 893
 894         INIT_WORK(&freq_scale_down_work,
 895                   cpufreq_interactive_freq_down);
 896
 897         spin_lock_init(&up_cpumask_lock);
 898         spin_lock_init(&down_cpumask_lock);
 899         mutex_init(&set_speed_lock);
 900
 901         idle_notifier_register(&cpufreq_interactive_idle_nb);
 902         INIT_WORK(&inputopen.inputopen_work, cpufreq_interactive_input_open);
 903         return cpufreq_register_governor(&cpufreq_gov_interactive);
 904
 905 err_freeuptask:
 906         put_task_struct(up_task);
 907         return -ENOMEM;
 908 }
 909
 910 #ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_INTERACTIVE
 911 fs_initcall(cpufreq_interactive_init);
 912 #else
 913 module_init(cpufreq_interactive_init);
 914 #endif
 915
 916 static void __exit cpufreq_interactive_exit(void)
 917 {
 918         cpufreq_unregister_governor(&cpufreq_gov_interactive);
 919         kthread_stop(up_task);
 920         put_task_struct(up_task);
 921         destroy_workqueue(down_wq);
 922 }
 923
 924 module_exit(cpufreq_interactive_exit);
 925
 926 MODULE_AUTHOR("Mike Chan <mike@android.com>");
 927 MODULE_DESCRIPTION("'cpufreq_interactive' - A cpufreq governor for "
 928         "Latency sensitive workloads");
 929 MODULE_LICENSE("GPL");