sched/numa: Add infrastructure for split shared/private accounting of NUMA hinting...

[firefly-linux-kernel-4.4.55.git] / kernel / sched / fair.c

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 8b15e9e1d1b889b51c37ae10b9e579b12ff78936..89eeb89fd99a4e7b33b7110a137085f7d94fc531 100644 (file)
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -886,6 +886,20 @@ static unsigned int task_scan_max(struct task_struct *p)
  */
 unsigned int sysctl_numa_balancing_settle_count __read_mostly = 3;
 
+static inline int task_faults_idx(int nid, int priv)
+{
+       return 2 * nid + priv;
+}
+
+static inline unsigned long task_faults(struct task_struct *p, int nid)
+{
+       if (!p->numa_faults)
+               return 0;
+
+       return p->numa_faults[task_faults_idx(nid, 0)] +
+               p->numa_faults[task_faults_idx(nid, 1)];
+}
+
 static unsigned long weighted_cpuload(const int cpu);
 
 
@@ -928,13 +942,19 @@ static void task_numa_placement(struct task_struct *p)
        /* Find the node with the highest number of faults */
        for_each_online_node(nid) {
                unsigned long faults;
+               int priv, i;
 
-               /* Decay existing window and copy faults since last scan */
-               p->numa_faults[nid] >>= 1;
-               p->numa_faults[nid] += p->numa_faults_buffer[nid];
-               p->numa_faults_buffer[nid] = 0;
+               for (priv = 0; priv < 2; priv++) {
+                       i = task_faults_idx(nid, priv);
 
-               faults = p->numa_faults[nid];
+                       /* Decay existing window, copy faults since last scan */
+                       p->numa_faults[i] >>= 1;
+                       p->numa_faults[i] += p->numa_faults_buffer[i];
+                       p->numa_faults_buffer[i] = 0;
+               }
+
+               /* Find maximum private faults */
+               faults = p->numa_faults[task_faults_idx(nid, 1)];
                if (faults > max_faults) {
                        max_faults = faults;
                        max_nid = nid;
@@ -970,16 +990,20 @@ static void task_numa_placement(struct task_struct *p)
 /*
  * Got a PROT_NONE fault for a page on @node.
  */
-void task_numa_fault(int node, int pages, bool migrated)
+void task_numa_fault(int last_nid, int node, int pages, bool migrated)
 {
        struct task_struct *p = current;
+       int priv;
 
        if (!numabalancing_enabled)
                return;
 
+       /* For now, do not attempt to detect private/shared accesses */
+       priv = 1;
+
        /* Allocate buffer to track faults on a per-node basis */
        if (unlikely(!p->numa_faults)) {
-               int size = sizeof(*p->numa_faults) * nr_node_ids;
+               int size = sizeof(*p->numa_faults) * 2 * nr_node_ids;
 
                /* numa_faults and numa_faults_buffer share the allocation */
                p->numa_faults = kzalloc(size * 2, GFP_KERNEL|__GFP_NOWARN);
@@ -987,7 +1011,7 @@ void task_numa_fault(int node, int pages, bool migrated)
                        return;
 
                BUG_ON(p->numa_faults_buffer);
-               p->numa_faults_buffer = p->numa_faults + nr_node_ids;
+               p->numa_faults_buffer = p->numa_faults + (2 * nr_node_ids);
        }
 
        /*
@@ -1005,7 +1029,7 @@ void task_numa_fault(int node, int pages, bool migrated)
 
        task_numa_placement(p);
 
-       p->numa_faults_buffer[node] += pages;
+       p->numa_faults_buffer[task_faults_idx(node, priv)] += pages;
 }
 
 static void reset_ptenuma_scan(struct task_struct *p)
@@ -4146,7 +4170,7 @@ static bool migrate_improves_locality(struct task_struct *p, struct lb_env *env)
                return false;
 
        if (dst_nid == p->numa_preferred_nid ||
-           p->numa_faults[dst_nid] > p->numa_faults[src_nid])
+           task_faults(p, dst_nid) > task_faults(p, src_nid))
                return true;
 
        return false;
@@ -4170,7 +4194,7 @@ static bool migrate_degrades_locality(struct task_struct *p, struct lb_env *env)
            p->numa_migrate_seq >= sysctl_numa_balancing_settle_count)
                return false;
 
-       if (p->numa_faults[dst_nid] < p->numa_faults[src_nid])
+       if (task_faults(p, dst_nid) < task_faults(p, src_nid))
                return true;
 
        return false;