sched: implement usage tracking

With the frame-work for runnable tracking now fully in place.  Per-entity usage
tracking is a simple and low-overhead addition.

Signed-off-by: Paul Turner <pjt@google.com>
Reviewed-by: Ben Segall <bsegall@google.com>

diff --git a/include/linux/sched.h b/include/linux/sched.h
index 178a8d909f14a3dcdcbc0ce255572975c8b3b221..fe752ca19a8dfbc19033e11627209a08151bbe6b 100644 (file)
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -931,6 +931,7 @@ struct sched_avg {
        u64 last_runnable_update;
        s64 decay_count;
        unsigned long load_avg_contrib;
+       u32 usage_avg_sum;
 };
 
 #ifdef CONFIG_SCHEDSTATS

diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c
index 75024a673520b9c5e0e506b1e0963d782cc84c5d..fbd8caa83ef5b09ea6a21ee169ceb7459aead978 100644 (file)
--- a/kernel/sched/debug.c
+++ b/kernel/sched/debug.c
@@ -94,6 +94,7 @@ static void print_cfs_group_stats(struct seq_file *m, int cpu, struct task_group
 #ifdef CONFIG_SMP
        P(se->avg.runnable_avg_sum);
        P(se->avg.runnable_avg_period);
+       P(se->avg.usage_avg_sum);
        P(se->avg.load_avg_contrib);
        P(se->avg.decay_count);
 #endif
@@ -223,6 +224,8 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq)
                        cfs_rq->tg_runnable_contrib);
        SEQ_printf(m, "  .%-30s: %d\n", "tg->runnable_avg",
                        atomic_read(&cfs_rq->tg->runnable_avg));
+       SEQ_printf(m, "  .%-30s: %d\n", "tg->usage_avg",
+                       atomic_read(&cfs_rq->tg->usage_avg));
 #endif
 
        print_cfs_group_stats(m, cpu, cfs_rq->tg);

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index c61a614465c8ebf13b5f71aabf23a78c9e8533a6..98979ddf5407bb1318e13c29ef761781afde2485 100644 (file)
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -1231,7 +1231,8 @@ static u32 __compute_runnable_contrib(u64 n)
  */
 static __always_inline int __update_entity_runnable_avg(u64 now,
                                                        struct sched_avg *sa,
-                                                       int runnable)
+                                                       int runnable,
+                                                       int running)
 {
        u64 delta, periods;
        u32 runnable_contrib;
@@ -1270,6 +1271,8 @@ static __always_inline int __update_entity_runnable_avg(u64 now,
                delta_w = 1024 - delta_w;
                if (runnable)
                        sa->runnable_avg_sum += delta_w;
+               if (running)
+                       sa->usage_avg_sum += delta_w;
                sa->runnable_avg_period += delta_w;
 
                delta -= delta_w;
@@ -1282,17 +1285,22 @@ static __always_inline int __update_entity_runnable_avg(u64 now,
                                                  periods + 1);
                sa->runnable_avg_period = decay_load(sa->runnable_avg_period,
                                                     periods + 1);
+               sa->usage_avg_sum = decay_load(sa->usage_avg_sum, periods + 1);
 
                /* Efficiently calculate \sum (1..n_period) 1024*y^i */
                runnable_contrib = __compute_runnable_contrib(periods);
                if (runnable)
                        sa->runnable_avg_sum += runnable_contrib;
+               if (running)
+                       sa->usage_avg_sum += runnable_contrib;
                sa->runnable_avg_period += runnable_contrib;
        }
 
        /* Remainder of delta accrued against u_0` */
        if (runnable)
                sa->runnable_avg_sum += delta;
+       if (running)
+               sa->usage_avg_sum += delta;
        sa->runnable_avg_period += delta;
 
        return decayed;
@@ -1338,16 +1346,28 @@ static inline void __update_tg_runnable_avg(struct sched_avg *sa,
                                                  struct cfs_rq *cfs_rq)
 {
        struct task_group *tg = cfs_rq->tg;
-       long contrib;
+       long contrib, usage_contrib;
 
        /* The fraction of a cpu used by this cfs_rq */
        contrib = div_u64(sa->runnable_avg_sum << NICE_0_SHIFT,
                          sa->runnable_avg_period + 1);
        contrib -= cfs_rq->tg_runnable_contrib;
 
-       if (abs(contrib) > cfs_rq->tg_runnable_contrib / 64) {
+       usage_contrib = div_u64(sa->usage_avg_sum << NICE_0_SHIFT,
+                               sa->runnable_avg_period + 1);
+       usage_contrib -= cfs_rq->tg_usage_contrib;
+
+       /*
+        * contrib/usage at this point represent deltas, only update if they
+        * are substantive.
+        */
+       if ((abs(contrib) > cfs_rq->tg_runnable_contrib / 64) ||
+           (abs(usage_contrib) > cfs_rq->tg_usage_contrib / 64)) {
                atomic_add(contrib, &tg->runnable_avg);
                cfs_rq->tg_runnable_contrib += contrib;
+
+               atomic_add(usage_contrib, &tg->usage_avg);
+               cfs_rq->tg_usage_contrib += usage_contrib;
        }
 }
 
@@ -1453,7 +1473,8 @@ static inline void update_entity_load_avg(struct sched_entity *se,
        else
                now = cfs_rq_clock_task(group_cfs_rq(se));
 
-       if (!__update_entity_runnable_avg(now, &se->avg, se->on_rq))
+       if (!__update_entity_runnable_avg(now, &se->avg, se->on_rq,
+                                         cfs_rq->curr == se))
                return;
 
        contrib_delta = __update_entity_load_avg_contrib(se);
@@ -1497,7 +1518,8 @@ static void update_cfs_rq_blocked_load(struct cfs_rq *cfs_rq, int force_update)
 
 static inline void update_rq_runnable_avg(struct rq *rq, int runnable)
 {
-       __update_entity_runnable_avg(rq->clock_task, &rq->avg, runnable);
+       __update_entity_runnable_avg(rq->clock_task, &rq->avg, runnable,
+                                    runnable);
        __update_tg_runnable_avg(&rq->avg, &rq->cfs);
 }
 
@@ -1886,6 +1908,7 @@ set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
                 */
                update_stats_wait_end(cfs_rq, se);
                __dequeue_entity(cfs_rq, se);
+               update_entity_load_avg(se, 1);
        }
 
        update_stats_curr_start(cfs_rq, se);

diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index ce39224d615599c40a1d6e7f8f4a78d728e2acad..e1d155c68dfc4e8ef61f95141e672e284059897d 100644 (file)
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -142,7 +142,7 @@ struct task_group {
 
        atomic_t load_weight;
        atomic64_t load_avg;
-       atomic_t runnable_avg;
+       atomic_t runnable_avg, usage_avg;
 #endif
 
 #ifdef CONFIG_RT_GROUP_SCHED
@@ -279,7 +279,7 @@ struct cfs_rq {
 #endif /* CONFIG_FAIR_GROUP_SCHED */
 /* These always depend on CONFIG_FAIR_GROUP_SCHED */
 #ifdef CONFIG_FAIR_GROUP_SCHED
-       u32 tg_runnable_contrib;
+       u32 tg_runnable_contrib, tg_usage_contrib;
        u64 tg_load_contrib;
 #endif /* CONFIG_FAIR_GROUP_SCHED */