sched: optimize cond_resched()

Optimize cond_resched() by removing one conditional.

Currently cond_resched() checks system_state ==
SYSTEM_RUNNING in order to avoid scheduling before the
scheduler is running.

We can however, as per suggestion of Matt, use
PREEMPT_ACTIVE to accomplish that very same.

Suggested-by: Matt Mackall <mpm@selenic.com>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Acked-by: Matt Mackall <mpm@selenic.com>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/include/linux/sched.h b/include/linux/sched.h
index 2a99f1c15cf80031ba66358d44ca4c472903162c..16a982e389fb289560d13039bfb608fd82bbdae2 100644 (file)
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -501,8 +501,11 @@ struct task_cputime {
 /*
  * Disable preemption until the scheduler is running.
  * Reset by start_kernel()->sched_init()->init_idle().
+ *
+ * We include PREEMPT_ACTIVE to avoid cond_resched() from working
+ * before the scheduler is active -- see should_resched().
  */
-#define INIT_PREEMPT_COUNT     (1)
+#define INIT_PREEMPT_COUNT     (1 + PREEMPT_ACTIVE)
 
 /**
  * struct thread_group_cputimer - thread group interval timer counts

diff --git a/kernel/sched.c b/kernel/sched.c
index 7c9098d186e6f8e398c0cb9500c1efa2120293e2..01f55ada3598a8e46dd25f7953660d69fc5258ee 100644 (file)
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -6541,6 +6541,11 @@ SYSCALL_DEFINE0(sched_yield)
        return 0;
 }
 
+static inline int should_resched(void)
+{
+       return need_resched() && !(preempt_count() & PREEMPT_ACTIVE);
+}
+
 static void __cond_resched(void)
 {
 #ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
@@ -6560,8 +6565,7 @@ static void __cond_resched(void)
 
 int __sched _cond_resched(void)
 {
-       if (need_resched() && !(preempt_count() & PREEMPT_ACTIVE) &&
-                                       system_state == SYSTEM_RUNNING) {
+       if (should_resched()) {
                __cond_resched();
                return 1;
        }
@@ -6579,12 +6583,12 @@ EXPORT_SYMBOL(_cond_resched);
  */
 int cond_resched_lock(spinlock_t *lock)
 {
-       int resched = need_resched() && system_state == SYSTEM_RUNNING;
+       int resched = should_resched();
        int ret = 0;
 
        if (spin_needbreak(lock) || resched) {
                spin_unlock(lock);
-               if (resched && need_resched())
+               if (resched)
                        __cond_resched();
                else
                        cpu_relax();
@@ -6599,7 +6603,7 @@ int __sched cond_resched_softirq(void)
 {
        BUG_ON(!in_softirq());
 
-       if (need_resched() && system_state == SYSTEM_RUNNING) {
+       if (should_resched()) {
                local_bh_enable();
                __cond_resched();
                local_bh_disable();