cpuquiet: Runnable threads governor

[perf] The runnable threads governor only looks at the average number of
runnables in the system to make a decision when bringing cores
offline/online. First pass; tweaks thresholds and delays to reduce
decision latency to about ~50-70ms per core (from ~100-150ms per core)

Change-Id: Idd3b268a74a8f56ad3fc0e5c7f388174d1b6611f
Signed-off-by: Sai Charan Gurrappadi <sgurrappadi@nvidia.com>
Reviewed-on: http://git-master/r/124679
Reviewed-by: Automatic_Commit_Validation_User
Reviewed-by: Diwakar Tundlam <dtundlam@nvidia.com>

diff --git a/drivers/cpuquiet/governors/Makefile b/drivers/cpuquiet/governors/Makefile
index c70803127082f37709e388c6afd9be5505c5238e..e199d73008f1821f2d61bced6216a6464c9e9b42 100644 (file)
--- a/drivers/cpuquiet/governors/Makefile
+++ b/drivers/cpuquiet/governors/Makefile
@@ -1 +1 @@
-obj-y += userspace.o balanced.o
+obj-y += userspace.o balanced.o runnable_threads.o

diff --git a/drivers/cpuquiet/governors/runnable_threads.c b/drivers/cpuquiet/governors/runnable_threads.c
new file mode 100644 (file)
index 0000000..a44abe9
--- /dev/null
+++ b/drivers/cpuquiet/governors/runnable_threads.c
@@ -0,0 +1,247 @@
+/*
+ * Copyright (c) 2012 NVIDIA CORPORATION.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 of the License.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ */
+
+#include <linux/kernel.h>
+#include <linux/cpuquiet.h>
+#include <linux/cpumask.h>
+#include <linux/module.h>
+#include <linux/pm_qos.h>
+#include <linux/jiffies.h>
+#include <linux/slab.h>
+#include <linux/cpu.h>
+#include <linux/sched.h>
+
+typedef enum {
+       DISABLED,
+       IDLE,
+       DOWN,
+       UP,
+} RUNNABLES_STATE;
+
+static struct delayed_work runnables_work;
+static struct kobject *runnables_kobject;
+
+/* configurable parameters */
+static unsigned int sample_rate = 20;          /* msec */
+
+static RUNNABLES_STATE runnables_state;
+static struct workqueue_struct *runnables_wq;
+
+#define NR_FSHIFT_EXP  3
+#define NR_FSHIFT      (1 << NR_FSHIFT_EXP)
+/* avg run threads * 8 (e.g., 11 = 1.375 threads) */
+static unsigned int default_thresholds[] = {
+       9, 17, 25, UINT_MAX
+};
+
+static unsigned int nr_run_last;
+static unsigned int nr_run_hysteresis = 4;             /* 1 / 4 thread */
+static unsigned int default_threshold_level = 4;       /* 1 / 4 thread */
+static unsigned int nr_run_thresholds[NR_CPUS];
+
+DEFINE_MUTEX(runnables_work_lock);
+
+static void update_runnables_state(void)
+{
+       unsigned int nr_cpus = num_online_cpus();
+       int max_cpus = pm_qos_request(PM_QOS_MAX_ONLINE_CPUS) ? : 4;
+       int min_cpus = pm_qos_request(PM_QOS_MIN_ONLINE_CPUS);
+       unsigned int avg_nr_run = avg_nr_running();
+       unsigned int nr_run;
+
+       if (runnables_state == DISABLED)
+               return;
+
+       for (nr_run = 1; nr_run < ARRAY_SIZE(nr_run_thresholds); nr_run++) {
+               unsigned int nr_threshold = nr_run_thresholds[nr_run - 1];
+               if (nr_run_last <= nr_run)
+                       nr_threshold += NR_FSHIFT / nr_run_hysteresis;
+               if (avg_nr_run <= (nr_threshold << (FSHIFT - NR_FSHIFT_EXP)))
+                       break;
+       }
+       nr_run_last = nr_run;
+
+       if ((nr_cpus > max_cpus || nr_run < nr_cpus) && nr_cpus >= min_cpus) {
+               runnables_state = DOWN;
+       } else if (nr_cpus < min_cpus || nr_run > nr_cpus) {
+               runnables_state =  UP;
+       } else {
+               runnables_state = IDLE;
+       }
+}
+
+static unsigned int get_lightest_loaded_cpu_n(void)
+{
+       unsigned long min_avg_runnables = ULONG_MAX;
+       unsigned int cpu = nr_cpu_ids;
+       int i;
+
+       for_each_online_cpu(i) {
+               unsigned int nr_runnables = get_avg_nr_running(i);
+
+               if (i > 0 && min_avg_runnables > nr_runnables) {
+                       cpu = i;
+                       min_avg_runnables = nr_runnables;
+               }
+       }
+
+       return cpu;
+}
+
+static void runnables_work_func(struct work_struct *work)
+{
+       bool up = false;
+       bool sample = false;
+       unsigned int cpu = nr_cpu_ids;
+
+       mutex_lock(&runnables_work_lock);
+
+       update_runnables_state();
+
+       switch (runnables_state) {
+       case DISABLED:
+               break;
+       case IDLE:
+               sample = true;
+               break;
+       case UP:
+               cpu = cpumask_next_zero(0, cpu_online_mask);
+               up = true;
+               sample = true;
+               break;
+       case DOWN:
+               cpu = get_lightest_loaded_cpu_n();
+               sample = true;
+               break;
+       default:
+               pr_err("%s: invalid cpuquiet runnable governor state %d\n",
+                       __func__, runnables_state);
+               break;
+       }
+
+       if (sample)
+               queue_delayed_work(runnables_wq, &runnables_work,
+                                       msecs_to_jiffies(sample_rate));
+
+       if (cpu < nr_cpu_ids) {
+               if (up)
+                       cpuquiet_wake_cpu(cpu);
+               else
+                       cpuquiet_quiesence_cpu(cpu);
+       }
+
+       mutex_unlock(&runnables_work_lock);
+}
+
+CPQ_BASIC_ATTRIBUTE(sample_rate, 0644, uint);
+CPQ_BASIC_ATTRIBUTE(nr_run_hysteresis, 0644, uint);
+
+static struct attribute *runnables_attributes[] = {
+       &sample_rate_attr.attr,
+       &nr_run_hysteresis_attr.attr,
+       NULL,
+};
+
+static const struct sysfs_ops runnables_sysfs_ops = {
+       .show = cpuquiet_auto_sysfs_show,
+       .store = cpuquiet_auto_sysfs_store,
+};
+
+static struct kobj_type ktype_runnables = {
+       .sysfs_ops = &runnables_sysfs_ops,
+       .default_attrs = runnables_attributes,
+};
+
+static int runnables_sysfs(void)
+{
+       int err;
+
+       runnables_kobject = kzalloc(sizeof(*runnables_kobject),
+                               GFP_KERNEL);
+
+       if (!runnables_kobject)
+               return -ENOMEM;
+
+       err = cpuquiet_kobject_init(runnables_kobject, &ktype_runnables,
+                               "runnable_threads");
+
+       if (err)
+               kfree(runnables_kobject);
+
+       return err;
+}
+
+static void runnables_stop(void)
+{
+       runnables_state = DISABLED;
+       cancel_delayed_work_sync(&runnables_work);
+       destroy_workqueue(runnables_wq);
+       kobject_put(runnables_kobject);
+}
+
+static int runnables_start(void)
+{
+       int err, i;
+
+       err = runnables_sysfs();
+       if (err)
+               return err;
+
+       runnables_wq = alloc_workqueue("cpuquiet-runnables",
+                       WQ_UNBOUND | WQ_RESCUER | WQ_FREEZABLE, 1);
+       if (!runnables_wq)
+               return -ENOMEM;
+
+       INIT_DELAYED_WORK(&runnables_work, runnables_work_func);
+
+       for(i = 0; i < ARRAY_SIZE(nr_run_thresholds); ++i) {
+               if (i < ARRAY_SIZE(default_thresholds))
+                       nr_run_thresholds[i] = default_thresholds[i];
+               else if (i == (ARRAY_SIZE(nr_run_thresholds) - 1))
+                       nr_run_thresholds[i] = UINT_MAX;
+               else
+                       nr_run_thresholds[i] = i + 1 +
+                               NR_FSHIFT / default_threshold_level;
+       }
+
+       runnables_state = IDLE;
+       runnables_work_func(NULL);
+
+       return 0;
+}
+
+struct cpuquiet_governor runnables_governor = {
+       .name                     = "runnable",
+       .start                    = runnables_start,
+       .stop                     = runnables_stop,
+       .owner                    = THIS_MODULE,
+};
+
+static int __init init_runnables(void)
+{
+       return cpuquiet_register_governor(&runnables_governor);
+}
+
+static void __exit exit_runnables(void)
+{
+       cpuquiet_unregister_governor(&runnables_governor);
+}
+
+MODULE_LICENSE("GPL");
+module_init(init_runnables);
+module_exit(exit_runnables);