FIX: sched/tune: move schedtune_nornalize_energy into fair.c

The energy normalization function is required to get the proper values
for the P-E space filtering function to work.
That normalization is part of the hot wakeup path and currently implemented
with a function call.

Moving the normalization function into fair.c allows the compiler to
further optimize that code by reducing overheads in the wakeup hot path.

Signed-off-by: Patrick Bellasi <patrick.bellasi@arm.com>
[jstultz: fwdported to 4.4]
Signed-off-by: John Stultz <john.stultz@linaro.org>

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 08dad006801c3f1712b690cb5264f9f841789e3c..f8f5529dcae8d0ac92c2721ee80be1ed4952e1a6 100644 (file)
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -4996,44 +4996,6 @@ static inline bool cpu_in_sg(struct sched_group *sg, int cpu)
        return cpu != -1 && cpumask_test_cpu(cpu, sched_group_cpus(sg));
 }
 
-#ifdef CONFIG_SCHED_TUNE
-static int energy_diff_evaluate(struct energy_env *eenv)
-{
-       unsigned int boost;
-       int nrg_delta;
-
-       /* Return energy diff when boost margin is 0 */
-#ifdef CONFIG_CGROUP_SCHEDTUNE
-       boost = schedtune_task_boost(eenv->task);
-#else
-       boost = get_sysctl_sched_cfs_boost();
-#endif
-       if (boost == 0)
-               return eenv->nrg.diff;
-
-       /* Compute normalized energy diff */
-       nrg_delta = schedtune_normalize_energy(eenv->nrg.diff);
-       eenv->nrg.delta = nrg_delta;
-
-       eenv->payoff = schedtune_accept_deltas(
-                       eenv->nrg.delta,
-                       eenv->cap.delta,
-                       eenv->task);
-
-       /*
-        * When SchedTune is enabled, the energy_diff() function will return
-        * the computed energy payoff value. Since the energy_diff() return
-        * value is expected to be negative by its callers, this evaluation
-        * function return a negative value each time the evaluation return a
-        * positive payoff, which is the condition for the acceptance of
-        * a scheduling decision
-        */
-       return -eenv->payoff;
-}
-#else /* CONFIG_SCHED_TUNE */
-#define energy_diff_evaluate(eenv) eenv->nrg.diff
-#endif
-
 /*
  * energy_diff(): Estimate the energy impact of changing the utilization
  * distribution. eenv specifies the change: utilisation amount, source, and
@@ -5041,12 +5003,11 @@ static int energy_diff_evaluate(struct energy_env *eenv)
  * utilization is removed from or added to the system (e.g. task wake-up). If
  * both are specified, the utilization is migrated.
  */
-static int energy_diff(struct energy_env *eenv)
+static inline int __energy_diff(struct energy_env *eenv)
 {
        struct sched_domain *sd;
        struct sched_group *sg;
        int sd_cpu = -1, energy_before = 0, energy_after = 0;
-       int result;
 
        struct energy_env eenv_before = {
                .util_delta     = 0,
@@ -5090,16 +5051,90 @@ static int energy_diff(struct energy_env *eenv)
        eenv->nrg.diff = eenv->nrg.after - eenv->nrg.before;
        eenv->payoff = 0;
 
-       result = energy_diff_evaluate(eenv);
-
        trace_sched_energy_diff(eenv->task,
                        eenv->src_cpu, eenv->dst_cpu, eenv->util_delta,
                        eenv->nrg.before, eenv->nrg.after, eenv->nrg.diff,
                        eenv->cap.before, eenv->cap.after, eenv->cap.delta,
                        eenv->nrg.delta, eenv->payoff);
 
-       return result;
+       return eenv->nrg.diff;
+}
+
+#ifdef CONFIG_SCHED_TUNE
+
+struct target_nrg schedtune_target_nrg;
+
+/*
+ * System energy normalization
+ * Returns the normalized value, in the range [0..SCHED_LOAD_SCALE],
+ * corresponding to the specified energy variation.
+ */
+static inline int
+normalize_energy(int energy_diff)
+{
+       u32 normalized_nrg;
+#ifdef CONFIG_SCHED_DEBUG
+       int max_delta;
+
+       /* Check for boundaries */
+       max_delta  = schedtune_target_nrg.max_power;
+       max_delta -= schedtune_target_nrg.min_power;
+       WARN_ON(abs(energy_diff) >= max_delta);
+#endif
+
+       /* Do scaling using positive numbers to increase the range */
+       normalized_nrg = (energy_diff < 0) ? -energy_diff : energy_diff;
+
+       /* Scale by energy magnitude */
+       normalized_nrg <<= SCHED_LOAD_SHIFT;
+
+       /* Normalize on max energy for target platform */
+       normalized_nrg = reciprocal_divide(
+                       normalized_nrg, schedtune_target_nrg.rdiv);
+
+       return (energy_diff < 0) ? -normalized_nrg : normalized_nrg;
+}
+
+static inline int
+energy_diff(struct energy_env *eenv)
+{
+       unsigned int boost;
+       int nrg_delta;
+
+       /* Conpute "absolute" energy diff */
+       __energy_diff(eenv);
+
+       /* Return energy diff when boost margin is 0 */
+#ifdef CONFIG_CGROUP_SCHEDTUNE
+       boost = schedtune_task_boost(eenv->task);
+#else
+       boost = get_sysctl_sched_cfs_boost();
+#endif
+       if (boost == 0)
+               return eenv->nrg.diff;
+
+       /* Compute normalized energy diff */
+       nrg_delta = normalize_energy(eenv->nrg.diff);
+       eenv->nrg.delta = nrg_delta;
+
+       eenv->payoff = schedtune_accept_deltas(
+                       eenv->nrg.delta,
+                       eenv->cap.delta,
+                       eenv->task);
+
+       /*
+        * When SchedTune is enabled, the energy_diff() function will return
+        * the computed energy payoff value. Since the energy_diff() return
+        * value is expected to be negative by its callers, this evaluation
+        * function return a negative value each time the evaluation return a
+        * positive payoff, which is the condition for the acceptance of
+        * a scheduling decision
+        */
+       return -eenv->payoff;
 }
+#else /* CONFIG_SCHED_TUNE */
+#define energy_diff(eenv) __energy_diff(eenv)
+#endif
 
 /*
  * Detect M:N waker/wakee relationships via a switching-frequency heuristic.

diff --git a/kernel/sched/tune.c b/kernel/sched/tune.c
index b40d40dc3c49a4bd5227b09989cb8bb6c41af163..8ca8db2de818322f672af067b768bc0fc59ed86e 100644 (file)
--- a/kernel/sched/tune.c
+++ b/kernel/sched/tune.c
@@ -3,24 +3,17 @@
 #include <linux/kernel.h>
 #include <linux/percpu.h>
 #include <linux/printk.h>
-#include <linux/reciprocal_div.h>
 #include <linux/rcupdate.h>
 #include <linux/slab.h>
 
 #include <trace/events/sched.h>
 
 #include "sched.h"
+#include "tune.h"
 
 unsigned int sysctl_sched_cfs_boost __read_mostly;
 
-/*
- * System energy normalization constants
- */
-static struct target_nrg {
-       unsigned long min_power;
-       unsigned long max_power;
-       struct reciprocal_value rdiv;
-} schedtune_target_nrg;
+extern struct target_nrg schedtune_target_nrg;
 
 /* Performance Boost region (B) threshold params */
 static int perf_boost_idx;
@@ -587,37 +580,6 @@ sysctl_sched_cfs_boost_handler(struct ctl_table *table, int write,
        return 0;
 }
 
-/*
- * System energy normalization
- * Returns the normalized value, in the range [0..SCHED_LOAD_SCALE],
- * corresponding to the specified energy variation.
- */
-int
-schedtune_normalize_energy(int energy_diff)
-{
-       u32 normalized_nrg;
-       int max_delta;
-
-#ifdef CONFIG_SCHED_DEBUG
-       /* Check for boundaries */
-       max_delta  = schedtune_target_nrg.max_power;
-       max_delta -= schedtune_target_nrg.min_power;
-       WARN_ON(abs(energy_diff) >= max_delta);
-#endif
-
-       /* Do scaling using positive numbers to increase the range */
-       normalized_nrg = (energy_diff < 0) ? -energy_diff : energy_diff;
-
-       /* Scale by energy magnitude */
-       normalized_nrg <<= SCHED_LOAD_SHIFT;
-
-       /* Normalize on max energy for target platform */
-       normalized_nrg = reciprocal_divide(
-                       normalized_nrg, schedtune_target_nrg.rdiv);
-
-       return (energy_diff < 0) ? -normalized_nrg : normalized_nrg;
-}
-
 #ifdef CONFIG_SCHED_DEBUG
 static void
 schedtune_test_nrg(unsigned long delta_pwr)

diff --git a/kernel/sched/tune.h b/kernel/sched/tune.h
index f7273a5d994aa4708c74ad5d815fe984a2ef1a20..7d2aa7951554936214bace99822c0210141c25ef 100644 (file)
--- a/kernel/sched/tune.h
+++ b/kernel/sched/tune.h
@@ -1,6 +1,17 @@
 
 #ifdef CONFIG_SCHED_TUNE
 
+#include <linux/reciprocal_div.h>
+
+/*
+ * System energy normalization constants
+ */
+struct target_nrg {
+       unsigned long min_power;
+       unsigned long max_power;
+       struct reciprocal_value rdiv;
+};
+
 #ifdef CONFIG_CGROUP_SCHEDTUNE
 
 int schedtune_cpu_boost(int cpu);
@@ -25,7 +36,6 @@ int schedtune_accept_deltas(int nrg_delta, int cap_delta,
 #define schedtune_enqueue_task(task, cpu) do { } while (0)
 #define schedtune_dequeue_task(task, cpu) do { } while (0)
 
-#define schedtune_normalize_energy(energy) energy
 #define schedtune_accept_deltas(nrg_delta, cap_delta, task) nrg_delta
 
 #endif /* CONFIG_SCHED_TUNE */