2 * Copyright (c) 2012 NVIDIA CORPORATION. All rights reserved.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; version 2 of the License.
8 * This program is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * You should have received a copy of the GNU General Public License along
14 * with this program; if not, write to the Free Software Foundation, Inc.,
15 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19 #include <linux/kernel.h>
20 #include <linux/cpuquiet.h>
21 #include <linux/cpumask.h>
22 #include <linux/module.h>
23 #include <linux/cpufreq.h>
24 #include <linux/pm_qos_params.h>
25 #include <linux/jiffies.h>
26 #include <linux/slab.h>
27 #include <linux/cpu.h>
28 #include <linux/sched.h>
29 #include <linux/tick.h>
30 #include <asm/cputime.h>
53 static DEFINE_PER_CPU(struct idle_info, idleinfo);
54 static DEFINE_PER_CPU(unsigned int, cpu_load);
56 static struct timer_list load_timer;
57 static bool load_timer_active;
58 struct balanced_attribute {
59 struct attribute attr;
60 ssize_t (*show)(struct balanced_attribute *attr, char *buf);
61 ssize_t (*store)(struct balanced_attribute *attr, const char *buf,
66 #define BALANCED_ATTRIBUTE(_name, _mode) \
67 static struct balanced_attribute _name ## _attr = { \
68 .attr = {.name = __stringify(_name), .mode = _mode }, \
69 .show = show_attribute, \
70 .store = store_attribute, \
74 /* configurable parameters */
75 static unsigned long balance_level = 75;
76 static unsigned long idle_bottom_freq;
77 static unsigned long idle_top_freq;
78 static unsigned long up_delay;
79 static unsigned long down_delay;
81 static struct workqueue_struct *balanced_wq;
82 static struct delayed_work balanced_work;
83 static BALANCED_STATE balanced_state;
84 static struct kobject *balanced_kobject;
86 static void calculate_load_timer(unsigned long data)
89 u64 idle_time, elapsed_time;
91 if (!load_timer_active)
94 for_each_online_cpu(i) {
95 struct idle_info *iinfo = &per_cpu(idleinfo, i);
96 unsigned int *load = &per_cpu(cpu_load, i);
98 iinfo->idle_last = iinfo->idle_current;
99 iinfo->last_timestamp = iinfo->timestamp;
100 iinfo->idle_current =
101 get_cpu_idle_time_us(i, &iinfo->timestamp);
102 elapsed_time = iinfo->timestamp - iinfo->last_timestamp;
104 idle_time = iinfo->idle_current - iinfo->idle_last;
106 do_div(idle_time, elapsed_time);
107 *load = 100 - idle_time;
109 mod_timer(&load_timer, jiffies + msecs_to_jiffies(100));
112 static void start_load_timer(void)
116 if (load_timer_active)
119 load_timer_active = true;
121 for_each_online_cpu(i) {
122 struct idle_info *iinfo = &per_cpu(idleinfo, i);
124 iinfo->idle_current =
125 get_cpu_idle_time_us(i, &iinfo->timestamp);
127 mod_timer(&load_timer, jiffies + msecs_to_jiffies(100));
130 static void stop_load_timer(void)
132 if (!load_timer_active)
135 load_timer_active = false;
136 del_timer(&load_timer);
139 static unsigned int get_slowest_cpu_n(void)
141 unsigned int cpu = nr_cpu_ids;
142 unsigned long minload = ULONG_MAX;
145 for_each_online_cpu(i) {
146 unsigned int *load = &per_cpu(cpu_load, i);
148 if ((i > 0) && (minload > *load)) {
157 static unsigned int cpu_highest_speed(void)
159 unsigned int maxload = 0;
162 for_each_online_cpu(i) {
163 unsigned int *load = &per_cpu(cpu_load, i);
165 maxload = max(maxload, *load);
171 static unsigned int count_slow_cpus(unsigned int limit)
173 unsigned int cnt = 0;
176 for_each_online_cpu(i) {
177 unsigned int *load = &per_cpu(cpu_load, i);
186 static CPU_SPEED_BALANCE balanced_speed_balance(void)
188 unsigned long highest_speed = cpu_highest_speed();
189 unsigned long balanced_speed = highest_speed * balance_level / 100;
190 unsigned long skewed_speed = balanced_speed / 2;
191 unsigned int nr_cpus = num_online_cpus();
192 unsigned int max_cpus = pm_qos_request(PM_QOS_MAX_ONLINE_CPUS) ? : 4;
194 /* balanced: freq targets for all CPUs are above 50% of highest speed
195 biased: freq target for at least one CPU is below 50% threshold
196 skewed: freq targets for at least 2 CPUs are below 25% threshold */
197 if (count_slow_cpus(skewed_speed) >= 2 || nr_cpus > max_cpus)
198 return CPU_SPEED_SKEWED;
200 if (count_slow_cpus(balanced_speed) >= 1 || nr_cpus == max_cpus)
201 return CPU_SPEED_BIASED;
203 return CPU_SPEED_BALANCED;
206 static void balanced_work_func(struct work_struct *work)
209 unsigned int cpu = nr_cpu_ids;
210 CPU_SPEED_BALANCE balance;
212 switch (balanced_state) {
216 cpu = get_slowest_cpu_n();
217 if (cpu < nr_cpu_ids) {
219 queue_delayed_work(balanced_wq,
220 &balanced_work, down_delay);
225 balance = balanced_speed_balance();
228 /* cpu speed is up and balanced - one more on-line */
229 case CPU_SPEED_BALANCED:
230 cpu = cpumask_next_zero(0, cpu_online_mask);
231 if (cpu < nr_cpu_ids)
234 /* cpu speed is up, but skewed - remove one core */
235 case CPU_SPEED_SKEWED:
236 cpu = get_slowest_cpu_n();
237 if (cpu < nr_cpu_ids)
240 /* cpu speed is up, but under-utilized - do nothing */
241 case CPU_SPEED_BIASED:
246 balanced_wq, &balanced_work, up_delay);
249 pr_err("%s: invalid cpuquiet balanced governor state %d\n",
250 __func__, balanced_state);
253 if (cpu < nr_cpu_ids) {
255 cpuquiet_wake_cpu(cpu);
257 cpuquiet_quiesence_cpu(cpu);
261 static int balanced_cpufreq_transition(struct notifier_block *nb,
262 unsigned long state, void *data)
264 struct cpufreq_freqs *freqs = data;
265 unsigned long cpu_freq;
267 if (state == CPUFREQ_POSTCHANGE || state == CPUFREQ_RESUMECHANGE) {
268 cpu_freq = freqs->new;
270 switch (balanced_state) {
272 if (cpu_freq > idle_top_freq) {
275 balanced_wq, &balanced_work, up_delay);
277 } else if (cpu_freq <= idle_bottom_freq) {
278 balanced_state = DOWN;
280 balanced_wq, &balanced_work,
286 if (cpu_freq > idle_top_freq) {
289 balanced_wq, &balanced_work, up_delay);
294 if (cpu_freq <= idle_bottom_freq) {
295 balanced_state = DOWN;
296 queue_delayed_work(balanced_wq,
297 &balanced_work, down_delay);
302 pr_err("%s: invalid cpuquiet balanced governor "
303 "state %d\n", __func__, balanced_state);
310 static struct notifier_block balanced_cpufreq_nb = {
311 .notifier_call = balanced_cpufreq_transition,
314 static ssize_t show_attribute(struct balanced_attribute *battr, char *buf)
316 return sprintf(buf, "%lu\n", *(battr->param));
319 static ssize_t store_attribute(struct balanced_attribute *battr,
320 const char *buf, size_t count)
325 err = strict_strtoul(buf, 0, &val);
329 *(battr->param) = val;
334 static ssize_t balanced_sysfs_store(struct kobject *kobj,
335 struct attribute *attr, const char *buf, size_t count)
337 struct balanced_attribute *battr =
338 container_of(attr, struct balanced_attribute, attr);
341 return battr->store(battr, buf, count);
346 static ssize_t balanced_sysfs_show(struct kobject *kobj,
347 struct attribute *attr, char *buf)
349 struct balanced_attribute *battr =
350 container_of(attr, struct balanced_attribute, attr);
352 return battr->show(battr, buf);
355 BALANCED_ATTRIBUTE(balance_level, 0644);
356 BALANCED_ATTRIBUTE(idle_bottom_freq, 0644);
357 BALANCED_ATTRIBUTE(idle_top_freq, 0644);
358 BALANCED_ATTRIBUTE(up_delay, 0644);
359 BALANCED_ATTRIBUTE(down_delay, 0644);
361 static struct attribute *balanced_attributes[] = {
362 &balance_level_attr.attr,
363 &idle_bottom_freq_attr.attr,
364 &idle_top_freq_attr.attr,
366 &down_delay_attr.attr,
370 static const struct sysfs_ops balanced_sysfs_ops = {
371 .show = balanced_sysfs_show,
372 .store = balanced_sysfs_store,
375 static struct kobj_type ktype_balanced = {
376 .sysfs_ops = &balanced_sysfs_ops,
377 .default_attrs = balanced_attributes,
380 static int balanced_sysfs(void)
384 balanced_kobject = kzalloc(sizeof(*balanced_kobject),
387 if (!balanced_kobject)
390 err = cpuquiet_kobject_init(balanced_kobject, &ktype_balanced,
394 kfree(balanced_kobject);
399 static void balanced_stop(void)
403 first unregister the notifiers. This ensures the governor state
404 can't be modified by a cpufreq transition
406 cpufreq_unregister_notifier(&balanced_cpufreq_nb,
407 CPUFREQ_TRANSITION_NOTIFIER);
409 /* now we can force the governor to be idle */
410 balanced_state = IDLE;
411 cancel_delayed_work_sync(&balanced_work);
412 destroy_workqueue(balanced_wq);
413 del_timer(&load_timer);
415 kobject_put(balanced_kobject);
418 static int balanced_start(void)
421 struct cpufreq_frequency_table *table;
423 err = balanced_sysfs();
427 balanced_wq = alloc_workqueue("cpuquiet-balanced",
428 WQ_UNBOUND | WQ_RESCUER | WQ_FREEZABLE, 1);
432 INIT_DELAYED_WORK(&balanced_work, balanced_work_func);
434 up_delay = msecs_to_jiffies(1000);
435 down_delay = msecs_to_jiffies(2000);
437 table = cpufreq_frequency_get_table(0);
438 for (count = 0; table[count].frequency != CPUFREQ_TABLE_END; count++)
441 idle_top_freq = table[(count / 2) - 1].frequency;
442 idle_bottom_freq = table[(count / 2) - 2].frequency;
444 cpufreq_register_notifier(&balanced_cpufreq_nb,
445 CPUFREQ_TRANSITION_NOTIFIER);
447 init_timer(&load_timer);
448 load_timer.function = calculate_load_timer;
453 struct cpuquiet_governor balanced_governor = {
455 .start = balanced_start,
456 .stop = balanced_stop,
457 .owner = THIS_MODULE,
460 static int __init init_balanced(void)
462 return cpuquiet_register_governor(&balanced_governor);
465 static void __exit exit_balanced(void)
467 cpuquiet_unregister_governor(&balanced_governor);
470 MODULE_LICENSE("GPL");
471 module_init(init_balanced);
472 module_exit(exit_balanced);