sched/core: Add first cpu w/ max/min orig capacity to root domain

This will allow to start iterating from a cpu with max or min original
capacity in the wakeup path regardless on which cpu the scheduler is
currently running (smp_processor_id()) or the previous cpu of the task
(task_cpu(p)). This iteration has to happen on a sched_domain spanning
all cpus in the order of the sched_groups of this sched_domain seen by
the starting cpu.

In case of an SMP system the first cpu with max orig capacity and the
the one with min orig capacity is the same. This can temporally happen
on a big.LITTLE system with hotplug as well.

E.g. the different order of cpu iteration can be used to map schedtune
task parameter 'boosted' into the cpu iteration order in
find_best_target().

Use of READ_ONCE()/WRITE_ONCE() to avoid load/store tearing.

Change-Id: I812fbd9c7e5f506617e456c0eec3edcd2c016e92
Signed-off-by: Dietmar Eggemann <dietmar.eggemann@arm.com>
(cherry picked from commit fd6e9543c1fd8971a5e2e68e39b2f6e591d46114)
Signed-off-by: Chris Redpath <chris.redpath@arm.com>

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 9efcfb3d0fc1ce80cf754a17bc86a9ecaa01d82d..495bc41907d6288122d7ab32428eb12f82324c97 100644 (file)
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -6155,6 +6155,9 @@ static int init_rootdomain(struct root_domain *rd)
                goto free_rto_mask;
 
        init_max_cpu_capacity(&rd->max_cpu_capacity);
+
+       rd->max_cap_orig_cpu = rd->min_cap_orig_cpu = -1;
+
        return 0;
 
 free_rto_mask:
@@ -7385,7 +7388,19 @@ static int build_sched_domains(const struct cpumask *cpu_map,
        /* Attach the domains */
        rcu_read_lock();
        for_each_cpu(i, cpu_map) {
+               int max_cpu = READ_ONCE(d.rd->max_cap_orig_cpu);
+               int min_cpu = READ_ONCE(d.rd->min_cap_orig_cpu);
+
+               if ((max_cpu < 0) || (cpu_rq(i)->cpu_capacity_orig >
+                   cpu_rq(max_cpu)->cpu_capacity_orig))
+                       WRITE_ONCE(d.rd->max_cap_orig_cpu, i);
+
+               if ((min_cpu < 0) || (cpu_rq(i)->cpu_capacity_orig <
+                   cpu_rq(min_cpu)->cpu_capacity_orig))
+                       WRITE_ONCE(d.rd->min_cap_orig_cpu, i);
+
                sd = *per_cpu_ptr(d.sd, i);
+
                cpu_attach_domain(sd, d.rd, i);
        }
        rcu_read_unlock();

diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 6bc2dd623b17c051e1b6d16136673cee3b41274a..2051fecdb9e58e0f87c32096173bc1bf609153c8 100644 (file)
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -561,6 +561,9 @@ struct root_domain {
 
        /* Maximum cpu capacity in the system. */
        struct max_cpu_capacity max_cpu_capacity;
+
+       /* First cpu with maximum and minimum original capacity */
+       int max_cap_orig_cpu, min_cap_orig_cpu;
 };
 
 extern struct root_domain def_root_domain;