sched: Fix select_idle_sibling() logic in select_task_rq_fair()

[firefly-linux-kernel-4.4.55.git] / kernel / sched_fair.c

diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
index d2c46e761fcd72edb6221e0e5415d55da3772079..01e311e6b47fd6b9368e49327f2467d8d148bc60 100644 (file)
--- a/kernel/sched_fair.c
+++ b/kernel/sched_fair.c
@@ -1250,6 +1250,7 @@ static int wake_affine(struct sched_domain *sd, struct task_struct *p, int sync)
         * effect of the currently running task from the load
         * of the current CPU:
         */
+       rcu_read_lock();
        if (sync) {
                tg = task_group(current);
                weight = current->se.load.weight;
@@ -1275,6 +1276,7 @@ static int wake_affine(struct sched_domain *sd, struct task_struct *p, int sync)
        balanced = !this_load ||
                100*(this_load + effective_load(tg, this_cpu, weight, weight)) <=
                imbalance*(load + effective_load(tg, prev_cpu, 0, weight));
+       rcu_read_unlock();
 
        /*
         * If the currently running task will sleep within
@@ -1381,6 +1383,56 @@ find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu)
        return idlest;
 }
 
+/*
+ * Try and locate an idle CPU in the sched_domain.
+ */
+static int select_idle_sibling(struct task_struct *p, int target)
+{
+       int cpu = smp_processor_id();
+       int prev_cpu = task_cpu(p);
+       struct sched_domain *sd;
+       int i;
+
+       /*
+        * If the task is going to be woken-up on this cpu and if it is
+        * already idle, then it is the right target.
+        */
+       if (target == cpu && idle_cpu(cpu))
+               return cpu;
+
+       /*
+        * If the task is going to be woken-up on the cpu where it previously
+        * ran and if it is currently idle, then it the right target.
+        */
+       if (target == prev_cpu && idle_cpu(prev_cpu))
+               return prev_cpu;
+
+       /*
+        * Otherwise, iterate the domains and find an elegible idle cpu.
+        */
+       for_each_domain(target, sd) {
+               if (!(sd->flags & SD_SHARE_PKG_RESOURCES))
+                       break;
+
+               for_each_cpu_and(i, sched_domain_span(sd), &p->cpus_allowed) {
+                       if (idle_cpu(i)) {
+                               target = i;
+                               break;
+                       }
+               }
+
+               /*
+                * Lets stop looking for an idle sibling when we reached
+                * the domain that spans the current cpu and prev_cpu.
+                */
+               if (cpumask_test_cpu(cpu, sched_domain_span(sd)) &&
+                   cpumask_test_cpu(prev_cpu, sched_domain_span(sd)))
+                       break;
+       }
+
+       return target;
+}
+
 /*
  * sched_balance_self: balance the current task (running on cpu) in domains
  * that have the 'flag' flag set. In practice, this is SD_BALANCE_FORK and
@@ -1438,38 +1490,14 @@ select_task_rq_fair(struct rq *rq, struct task_struct *p, int sd_flag, int wake_
                                want_sd = 0;
                }
 
-               if (want_affine && (tmp->flags & SD_WAKE_AFFINE)) {
-                       int candidate = -1, i;
-
-                       if (cpumask_test_cpu(prev_cpu, sched_domain_span(tmp)))
-                               candidate = cpu;
-
-                       /*
-                        * Check for an idle shared cache.
-                        */
-                       if (tmp->flags & SD_PREFER_SIBLING) {
-                               if (candidate == cpu) {
-                                       if (!cpu_rq(prev_cpu)->cfs.nr_running)
-                                               candidate = prev_cpu;
-                               }
-
-                               if (candidate == -1 || candidate == cpu) {
-                                       for_each_cpu(i, sched_domain_span(tmp)) {
-                                               if (!cpumask_test_cpu(i, &p->cpus_allowed))
-                                                       continue;
-                                               if (!cpu_rq(i)->cfs.nr_running) {
-                                                       candidate = i;
-                                                       break;
-                                               }
-                                       }
-                               }
-                       }
-
-                       if (candidate >= 0) {
-                               affine_sd = tmp;
-                               want_affine = 0;
-                               cpu = candidate;
-                       }
+               /*
+                * If both cpu and prev_cpu are part of this domain,
+                * cpu is a valid SD_WAKE_AFFINE target.
+                */
+               if (want_affine && (tmp->flags & SD_WAKE_AFFINE) &&
+                   cpumask_test_cpu(prev_cpu, sched_domain_span(tmp))) {
+                       affine_sd = tmp;
+                       want_affine = 0;
                }
 
                if (!want_sd && !want_affine)
@@ -1482,14 +1510,13 @@ select_task_rq_fair(struct rq *rq, struct task_struct *p, int sd_flag, int wake_
                        sd = tmp;
        }
 
+#ifdef CONFIG_FAIR_GROUP_SCHED
        if (sched_feat(LB_SHARES_UPDATE)) {
                /*
                 * Pick the largest domain to update shares over
                 */
                tmp = sd;
-               if (affine_sd && (!tmp ||
-                                 cpumask_weight(sched_domain_span(affine_sd)) >
-                                 cpumask_weight(sched_domain_span(sd))))
+               if (affine_sd && (!tmp || affine_sd->span_weight > sd->span_weight))
                        tmp = affine_sd;
 
                if (tmp) {
@@ -1498,9 +1525,14 @@ select_task_rq_fair(struct rq *rq, struct task_struct *p, int sd_flag, int wake_
                        spin_lock(&rq->lock);
                }
        }
+#endif
 
-       if (affine_sd && wake_affine(affine_sd, p, sync))
-               return cpu;
+       if (affine_sd) {
+               if (cpu == prev_cpu || wake_affine(affine_sd, p, sync))
+                       return select_idle_sibling(p, cpu);
+               else
+                       return select_idle_sibling(p, prev_cpu);
+       }
 
        while (sd) {
                int load_idx = sd->forkexec_idx;
@@ -1530,10 +1562,10 @@ select_task_rq_fair(struct rq *rq, struct task_struct *p, int sd_flag, int wake_
 
                /* Now try balancing at a lower domain level of new_cpu */
                cpu = new_cpu;
-               weight = cpumask_weight(sched_domain_span(sd));
+               weight = sd->span_weight;
                sd = NULL;
                for_each_domain(cpu, tmp) {
-                       if (weight <= cpumask_weight(sched_domain_span(tmp)))
+                       if (weight <= tmp->span_weight)
                                break;
                        if (tmp->flags & sd_flag)
                                sd = tmp;