2 * intel_powerclamp.c - package c-state idle injection
4 * Copyright (c) 2012, Intel Corporation.
7 * Arjan van de Ven <arjan@linux.intel.com>
8 * Jacob Pan <jacob.jun.pan@linux.intel.com>
10 * This program is free software; you can redistribute it and/or modify it
11 * under the terms and conditions of the GNU General Public License,
12 * version 2, as published by the Free Software Foundation.
14 * This program is distributed in the hope it will be useful, but WITHOUT
15 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
16 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
19 * You should have received a copy of the GNU General Public License along with
20 * this program; if not, write to the Free Software Foundation, Inc.,
21 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
25 * 1. better handle wakeup from external interrupts, currently a fixed
26 * compensation is added to clamping duration when excessive amount
27 * of wakeups are observed during idle time. the reason is that in
28 * case of external interrupts without need for ack, clamping down
29 * cpu in non-irq context does not reduce irq. for majority of the
30 * cases, clamping down cpu does help reduce irq as well, we should
31 * be able to differenciate the two cases and give a quantitative
32 * solution for the irqs that we can control. perhaps based on
33 * get_cpu_iowait_time_us()
35 * 2. synchronization with other hw blocks
40 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
42 #include <linux/module.h>
43 #include <linux/kernel.h>
44 #include <linux/delay.h>
45 #include <linux/kthread.h>
46 #include <linux/freezer.h>
47 #include <linux/cpu.h>
48 #include <linux/thermal.h>
49 #include <linux/slab.h>
50 #include <linux/tick.h>
51 #include <linux/debugfs.h>
52 #include <linux/seq_file.h>
53 #include <linux/sched/rt.h>
57 #include <asm/mwait.h>
58 #include <asm/cpu_device_id.h>
60 #include <asm/hardirq.h>
62 #define MAX_TARGET_RATIO (50U)
63 /* For each undisturbed clamping period (no extra wake ups during idle time),
64 * we increment the confidence counter for the given target ratio.
65 * CONFIDENCE_OK defines the level where runtime calibration results are
68 #define CONFIDENCE_OK (3)
69 /* Default idle injection duration, driver adjust sleep time to meet target
70 * idle ratio. Similar to frequency modulation.
72 #define DEFAULT_DURATION_JIFFIES (6)
74 static unsigned int target_mwait;
75 static struct dentry *debug_dir;
77 /* user selected target */
78 static unsigned int set_target_ratio;
79 static unsigned int current_ratio;
80 static bool should_skip;
81 static bool reduce_irq;
82 static atomic_t idle_wakeup_counter;
83 static unsigned int control_cpu; /* The cpu assigned to collect stat and update
84 * control parameters. default to BSP but BSP
90 static struct task_struct * __percpu *powerclamp_thread;
91 static struct thermal_cooling_device *cooling_dev;
92 static unsigned long *cpu_clamping_mask; /* bit map for tracking per cpu
96 static unsigned int duration;
97 static unsigned int pkg_cstate_ratio_cur;
98 static unsigned int window_size;
100 static int duration_set(const char *arg, const struct kernel_param *kp)
103 unsigned long new_duration;
105 ret = kstrtoul(arg, 10, &new_duration);
108 if (new_duration > 25 || new_duration < 6) {
109 pr_err("Out of recommended range %lu, between 6-25ms\n",
114 duration = clamp(new_duration, 6ul, 25ul);
122 static struct kernel_param_ops duration_ops = {
124 .get = param_get_int,
128 module_param_cb(duration, &duration_ops, &duration, 0644);
129 MODULE_PARM_DESC(duration, "forced idle time for each attempt in msec.");
131 struct powerclamp_calibration_data {
132 unsigned long confidence; /* used for calibration, basically a counter
133 * gets incremented each time a clamping
134 * period is completed without extra wakeups
135 * once that counter is reached given level,
136 * compensation is deemed usable.
138 unsigned long steady_comp; /* steady state compensation used when
139 * no extra wakeups occurred.
141 unsigned long dynamic_comp; /* compensate excessive wakeup from idle
142 * mostly from external interrupts.
146 static struct powerclamp_calibration_data cal_data[MAX_TARGET_RATIO];
148 static int window_size_set(const char *arg, const struct kernel_param *kp)
151 unsigned long new_window_size;
153 ret = kstrtoul(arg, 10, &new_window_size);
156 if (new_window_size > 10 || new_window_size < 2) {
157 pr_err("Out of recommended window size %lu, between 2-10\n",
162 window_size = clamp(new_window_size, 2ul, 10ul);
170 static struct kernel_param_ops window_size_ops = {
171 .set = window_size_set,
172 .get = param_get_int,
175 module_param_cb(window_size, &window_size_ops, &window_size, 0644);
176 MODULE_PARM_DESC(window_size, "sliding window in number of clamping cycles\n"
177 "\tpowerclamp controls idle ratio within this window. larger\n"
178 "\twindow size results in slower response time but more smooth\n"
179 "\tclamping results. default to 2.");
181 static void find_target_mwait(void)
183 unsigned int eax, ebx, ecx, edx;
184 unsigned int highest_cstate = 0;
185 unsigned int highest_subcstate = 0;
188 if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF)
191 cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &edx);
193 if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) ||
194 !(ecx & CPUID5_ECX_INTERRUPT_BREAK))
197 edx >>= MWAIT_SUBSTATE_SIZE;
198 for (i = 0; i < 7 && edx; i++, edx >>= MWAIT_SUBSTATE_SIZE) {
199 if (edx & MWAIT_SUBSTATE_MASK) {
201 highest_subcstate = edx & MWAIT_SUBSTATE_MASK;
204 target_mwait = (highest_cstate << MWAIT_SUBSTATE_SIZE) |
205 (highest_subcstate - 1);
209 static bool has_pkg_state_counter(void)
212 return !rdmsrl_safe(MSR_PKG_C2_RESIDENCY, &tmp) ||
213 !rdmsrl_safe(MSR_PKG_C3_RESIDENCY, &tmp) ||
214 !rdmsrl_safe(MSR_PKG_C6_RESIDENCY, &tmp) ||
215 !rdmsrl_safe(MSR_PKG_C7_RESIDENCY, &tmp);
218 static u64 pkg_state_counter(void)
229 if (!rdmsrl_safe(MSR_PKG_C2_RESIDENCY, &val))
236 if (!rdmsrl_safe(MSR_PKG_C3_RESIDENCY, &val))
243 if (!rdmsrl_safe(MSR_PKG_C6_RESIDENCY, &val))
250 if (!rdmsrl_safe(MSR_PKG_C7_RESIDENCY, &val))
259 static void noop_timer(unsigned long foo)
261 /* empty... just the fact that we get the interrupt wakes us up */
264 static unsigned int get_compensation(int ratio)
266 unsigned int comp = 0;
268 /* we only use compensation if all adjacent ones are good */
270 cal_data[ratio].confidence >= CONFIDENCE_OK &&
271 cal_data[ratio + 1].confidence >= CONFIDENCE_OK &&
272 cal_data[ratio + 2].confidence >= CONFIDENCE_OK) {
273 comp = (cal_data[ratio].steady_comp +
274 cal_data[ratio + 1].steady_comp +
275 cal_data[ratio + 2].steady_comp) / 3;
276 } else if (ratio == MAX_TARGET_RATIO - 1 &&
277 cal_data[ratio].confidence >= CONFIDENCE_OK &&
278 cal_data[ratio - 1].confidence >= CONFIDENCE_OK &&
279 cal_data[ratio - 2].confidence >= CONFIDENCE_OK) {
280 comp = (cal_data[ratio].steady_comp +
281 cal_data[ratio - 1].steady_comp +
282 cal_data[ratio - 2].steady_comp) / 3;
283 } else if (cal_data[ratio].confidence >= CONFIDENCE_OK &&
284 cal_data[ratio - 1].confidence >= CONFIDENCE_OK &&
285 cal_data[ratio + 1].confidence >= CONFIDENCE_OK) {
286 comp = (cal_data[ratio].steady_comp +
287 cal_data[ratio - 1].steady_comp +
288 cal_data[ratio + 1].steady_comp) / 3;
291 /* REVISIT: simple penalty of double idle injection */
294 /* do not exceed limit */
295 if (comp + ratio >= MAX_TARGET_RATIO)
296 comp = MAX_TARGET_RATIO - ratio - 1;
301 static void adjust_compensation(int target_ratio, unsigned int win)
304 struct powerclamp_calibration_data *d = &cal_data[target_ratio];
307 * adjust compensations if confidence level has not been reached or
308 * there are too many wakeups during the last idle injection period, we
309 * cannot trust the data for compensation.
311 if (d->confidence >= CONFIDENCE_OK ||
312 atomic_read(&idle_wakeup_counter) >
313 win * num_online_cpus())
316 delta = set_target_ratio - current_ratio;
317 /* filter out bad data */
318 if (delta >= 0 && delta <= (1+target_ratio/10)) {
321 roundup(delta+d->steady_comp, 2)/2;
323 d->steady_comp = delta;
328 static bool powerclamp_adjust_controls(unsigned int target_ratio,
329 unsigned int guard, unsigned int win)
331 static u64 msr_last, tsc_last;
332 u64 msr_now, tsc_now;
335 /* check result for the last window */
336 msr_now = pkg_state_counter();
339 /* calculate pkg cstate vs tsc ratio */
340 if (!msr_last || !tsc_last)
342 else if (tsc_now-tsc_last) {
343 val64 = 100*(msr_now-msr_last);
344 do_div(val64, (tsc_now-tsc_last));
345 current_ratio = val64;
352 adjust_compensation(target_ratio, win);
354 * too many external interrupts, set flag such
355 * that we can take measure later.
357 reduce_irq = atomic_read(&idle_wakeup_counter) >=
358 2 * win * num_online_cpus();
360 atomic_set(&idle_wakeup_counter, 0);
361 /* if we are above target+guard, skip */
362 return set_target_ratio + guard <= current_ratio;
365 static int clamp_thread(void *arg)
367 int cpunr = (unsigned long)arg;
368 DEFINE_TIMER(wakeup_timer, noop_timer, 0, 0);
369 static const struct sched_param param = {
370 .sched_priority = MAX_USER_RT_PRIO/2,
372 unsigned int count = 0;
373 unsigned int target_ratio;
375 set_bit(cpunr, cpu_clamping_mask);
377 init_timer_on_stack(&wakeup_timer);
378 sched_setscheduler(current, SCHED_FIFO, ¶m);
380 while (true == clamping && !kthread_should_stop() &&
383 unsigned long target_jiffies;
385 unsigned int compensation = 0;
386 int interval; /* jiffies to sleep for each attempt */
387 unsigned int duration_jiffies = msecs_to_jiffies(duration);
388 unsigned int window_size_now;
392 * make sure user selected ratio does not take effect until
393 * the next round. adjust target_ratio if user has changed
394 * target such that we can converge quickly.
396 target_ratio = set_target_ratio;
397 guard = 1 + target_ratio/20;
398 window_size_now = window_size;
402 * systems may have different ability to enter package level
403 * c-states, thus we need to compensate the injected idle ratio
404 * to achieve the actual target reported by the HW.
406 compensation = get_compensation(target_ratio);
407 interval = duration_jiffies*100/(target_ratio+compensation);
409 /* align idle time */
410 target_jiffies = roundup(jiffies, interval);
411 sleeptime = target_jiffies - jiffies;
414 schedule_timeout_interruptible(sleeptime);
416 * only elected controlling cpu can collect stats and update
417 * control parameters.
419 if (cpunr == control_cpu && !(count%window_size_now)) {
421 powerclamp_adjust_controls(target_ratio,
422 guard, window_size_now);
429 target_jiffies = jiffies + duration_jiffies;
430 mod_timer(&wakeup_timer, target_jiffies);
431 if (unlikely(local_softirq_pending()))
434 * stop tick sched during idle time, interrupts are still
435 * allowed. thus jiffies are updated properly.
438 /* mwait until target jiffies is reached */
439 while (time_before(jiffies, target_jiffies)) {
440 unsigned long ecx = 1;
441 unsigned long eax = target_mwait;
444 * REVISIT: may call enter_idle() to notify drivers who
445 * can save power during cpu idle. same for exit_idle()
448 stop_critical_timings();
449 mwait_idle_with_hints(eax, ecx);
450 start_critical_timings();
451 atomic_inc(&idle_wakeup_counter);
455 del_timer_sync(&wakeup_timer);
456 clear_bit(cpunr, cpu_clamping_mask);
462 * 1 HZ polling while clamping is active, useful for userspace
463 * to monitor actual idle ratio.
465 static void poll_pkg_cstate(struct work_struct *dummy);
466 static DECLARE_DELAYED_WORK(poll_pkg_cstate_work, poll_pkg_cstate);
467 static void poll_pkg_cstate(struct work_struct *dummy)
471 static unsigned long jiffies_last;
474 unsigned long jiffies_now;
478 msr_now = pkg_state_counter();
480 jiffies_now = jiffies;
482 /* calculate pkg cstate vs tsc ratio */
483 if (!msr_last || !tsc_last)
484 pkg_cstate_ratio_cur = 1;
486 if (tsc_now - tsc_last) {
487 val64 = 100 * (msr_now - msr_last);
488 do_div(val64, (tsc_now - tsc_last));
489 pkg_cstate_ratio_cur = val64;
495 jiffies_last = jiffies_now;
498 if (true == clamping)
499 schedule_delayed_work(&poll_pkg_cstate_work, HZ);
502 static int start_power_clamp(void)
505 struct task_struct *thread;
507 /* check if pkg cstate counter is completely 0, abort in this case */
508 if (!has_pkg_state_counter()) {
509 pr_err("pkg cstate counter not functional, abort\n");
513 set_target_ratio = clamp(set_target_ratio, 0U, MAX_TARGET_RATIO - 1);
514 /* prevent cpu hotplug */
519 if (!cpu_online(control_cpu))
520 control_cpu = smp_processor_id();
523 schedule_delayed_work(&poll_pkg_cstate_work, 0);
525 /* start one thread per online cpu */
526 for_each_online_cpu(cpu) {
527 struct task_struct **p =
528 per_cpu_ptr(powerclamp_thread, cpu);
530 thread = kthread_create_on_node(clamp_thread,
533 "kidle_inject/%ld", cpu);
534 /* bind to cpu here */
535 if (likely(!IS_ERR(thread))) {
536 kthread_bind(thread, cpu);
537 wake_up_process(thread);
547 static void end_power_clamp(void)
550 struct task_struct *thread;
554 * make clamping visible to other cpus and give per cpu clamping threads
555 * sometime to exit, or gets killed later.
559 if (bitmap_weight(cpu_clamping_mask, num_possible_cpus())) {
560 for_each_set_bit(i, cpu_clamping_mask, num_possible_cpus()) {
561 pr_debug("clamping thread for cpu %d alive, kill\n", i);
562 thread = *per_cpu_ptr(powerclamp_thread, i);
563 kthread_stop(thread);
568 static int powerclamp_cpu_callback(struct notifier_block *nfb,
569 unsigned long action, void *hcpu)
571 unsigned long cpu = (unsigned long)hcpu;
572 struct task_struct *thread;
573 struct task_struct **percpu_thread =
574 per_cpu_ptr(powerclamp_thread, cpu);
576 if (false == clamping)
581 thread = kthread_create_on_node(clamp_thread,
584 "kidle_inject/%lu", cpu);
585 if (likely(!IS_ERR(thread))) {
586 kthread_bind(thread, cpu);
587 wake_up_process(thread);
588 *percpu_thread = thread;
590 /* prefer BSP as controlling CPU */
597 if (test_bit(cpu, cpu_clamping_mask)) {
598 pr_err("cpu %lu dead but powerclamping thread is not\n",
600 kthread_stop(*percpu_thread);
602 if (cpu == control_cpu) {
603 control_cpu = smp_processor_id();
612 static struct notifier_block powerclamp_cpu_notifier = {
613 .notifier_call = powerclamp_cpu_callback,
616 static int powerclamp_get_max_state(struct thermal_cooling_device *cdev,
617 unsigned long *state)
619 *state = MAX_TARGET_RATIO;
624 static int powerclamp_get_cur_state(struct thermal_cooling_device *cdev,
625 unsigned long *state)
627 if (true == clamping)
628 *state = pkg_cstate_ratio_cur;
630 /* to save power, do not poll idle ratio while not clamping */
631 *state = -1; /* indicates invalid state */
636 static int powerclamp_set_cur_state(struct thermal_cooling_device *cdev,
637 unsigned long new_target_ratio)
641 new_target_ratio = clamp(new_target_ratio, 0UL,
642 (unsigned long) (MAX_TARGET_RATIO-1));
643 if (set_target_ratio == 0 && new_target_ratio > 0) {
644 pr_info("Start idle injection to reduce power\n");
645 set_target_ratio = new_target_ratio;
646 ret = start_power_clamp();
648 } else if (set_target_ratio > 0 && new_target_ratio == 0) {
649 pr_info("Stop forced idle injection\n");
650 set_target_ratio = 0;
652 } else /* adjust currently running */ {
653 set_target_ratio = new_target_ratio;
654 /* make new set_target_ratio visible to other cpus */
662 /* bind to generic thermal layer as cooling device*/
663 static struct thermal_cooling_device_ops powerclamp_cooling_ops = {
664 .get_max_state = powerclamp_get_max_state,
665 .get_cur_state = powerclamp_get_cur_state,
666 .set_cur_state = powerclamp_set_cur_state,
669 /* runs on Nehalem and later */
670 static const struct x86_cpu_id intel_powerclamp_ids[] = {
671 { X86_VENDOR_INTEL, 6, 0x1a},
672 { X86_VENDOR_INTEL, 6, 0x1c},
673 { X86_VENDOR_INTEL, 6, 0x1e},
674 { X86_VENDOR_INTEL, 6, 0x1f},
675 { X86_VENDOR_INTEL, 6, 0x25},
676 { X86_VENDOR_INTEL, 6, 0x26},
677 { X86_VENDOR_INTEL, 6, 0x2a},
678 { X86_VENDOR_INTEL, 6, 0x2c},
679 { X86_VENDOR_INTEL, 6, 0x2d},
680 { X86_VENDOR_INTEL, 6, 0x2e},
681 { X86_VENDOR_INTEL, 6, 0x2f},
682 { X86_VENDOR_INTEL, 6, 0x37},
683 { X86_VENDOR_INTEL, 6, 0x3a},
684 { X86_VENDOR_INTEL, 6, 0x3c},
685 { X86_VENDOR_INTEL, 6, 0x3d},
686 { X86_VENDOR_INTEL, 6, 0x3e},
687 { X86_VENDOR_INTEL, 6, 0x3f},
688 { X86_VENDOR_INTEL, 6, 0x45},
689 { X86_VENDOR_INTEL, 6, 0x46},
690 { X86_VENDOR_INTEL, 6, 0x4c},
691 { X86_VENDOR_INTEL, 6, 0x56},
694 MODULE_DEVICE_TABLE(x86cpu, intel_powerclamp_ids);
696 static int powerclamp_probe(void)
698 if (!x86_match_cpu(intel_powerclamp_ids)) {
699 pr_err("Intel powerclamp does not run on family %d model %d\n",
700 boot_cpu_data.x86, boot_cpu_data.x86_model);
703 if (!boot_cpu_has(X86_FEATURE_NONSTOP_TSC) ||
704 !boot_cpu_has(X86_FEATURE_CONSTANT_TSC) ||
705 !boot_cpu_has(X86_FEATURE_MWAIT) ||
706 !boot_cpu_has(X86_FEATURE_ARAT))
709 /* find the deepest mwait value */
715 static int powerclamp_debug_show(struct seq_file *m, void *unused)
719 seq_printf(m, "controlling cpu: %d\n", control_cpu);
720 seq_printf(m, "pct confidence steady dynamic (compensation)\n");
721 for (i = 0; i < MAX_TARGET_RATIO; i++) {
722 seq_printf(m, "%d\t%lu\t%lu\t%lu\n",
724 cal_data[i].confidence,
725 cal_data[i].steady_comp,
726 cal_data[i].dynamic_comp);
732 static int powerclamp_debug_open(struct inode *inode,
735 return single_open(file, powerclamp_debug_show, inode->i_private);
738 static const struct file_operations powerclamp_debug_fops = {
739 .open = powerclamp_debug_open,
742 .release = single_release,
743 .owner = THIS_MODULE,
746 static inline void powerclamp_create_debug_files(void)
748 debug_dir = debugfs_create_dir("intel_powerclamp", NULL);
752 if (!debugfs_create_file("powerclamp_calib", S_IRUGO, debug_dir,
753 cal_data, &powerclamp_debug_fops))
759 debugfs_remove_recursive(debug_dir);
762 static int powerclamp_init(void)
767 bitmap_size = BITS_TO_LONGS(num_possible_cpus()) * sizeof(long);
768 cpu_clamping_mask = kzalloc(bitmap_size, GFP_KERNEL);
769 if (!cpu_clamping_mask)
772 /* probe cpu features and ids here */
773 retval = powerclamp_probe();
777 /* set default limit, maybe adjusted during runtime based on feedback */
779 register_hotcpu_notifier(&powerclamp_cpu_notifier);
781 powerclamp_thread = alloc_percpu(struct task_struct *);
782 if (!powerclamp_thread) {
784 goto exit_unregister;
787 cooling_dev = thermal_cooling_device_register("intel_powerclamp", NULL,
788 &powerclamp_cooling_ops);
789 if (IS_ERR(cooling_dev)) {
791 goto exit_free_thread;
795 duration = jiffies_to_msecs(DEFAULT_DURATION_JIFFIES);
797 powerclamp_create_debug_files();
802 free_percpu(powerclamp_thread);
804 unregister_hotcpu_notifier(&powerclamp_cpu_notifier);
806 kfree(cpu_clamping_mask);
809 module_init(powerclamp_init);
811 static void powerclamp_exit(void)
813 unregister_hotcpu_notifier(&powerclamp_cpu_notifier);
815 free_percpu(powerclamp_thread);
816 thermal_cooling_device_unregister(cooling_dev);
817 kfree(cpu_clamping_mask);
819 cancel_delayed_work_sync(&poll_pkg_cstate_work);
820 debugfs_remove_recursive(debug_dir);
822 module_exit(powerclamp_exit);
824 MODULE_LICENSE("GPL");
825 MODULE_AUTHOR("Arjan van de Ven <arjan@linux.intel.com>");
826 MODULE_AUTHOR("Jacob Pan <jacob.jun.pan@linux.intel.com>");
827 MODULE_DESCRIPTION("Package Level C-state Idle Injection for Intel CPUs");