Merge branch 'for-3.17/core' of git://git.kernel.dk/linux-block

[firefly-linux-kernel-4.4.55.git] / kernel / locking / mutex.c

diff --git a/kernel/locking/mutex.c b/kernel/locking/mutex.c
index acca2c1a3c5e550a42cae2256e0b88b02f352faa..ae712b25e4922517a5fc13949d09f2c36294dadb 100644 (file)
--- a/kernel/locking/mutex.c
+++ b/kernel/locking/mutex.c
@@ -46,12 +46,6 @@
 # include <asm/mutex.h>
 #endif
 
-/*
- * A negative mutex count indicates that waiters are sleeping waiting for the
- * mutex.
- */
-#define        MUTEX_SHOW_NO_WAITER(mutex)     (atomic_read(&(mutex)->count) >= 0)
-
 void
 __mutex_init(struct mutex *lock, const char *name, struct lock_class_key *key)
 {
@@ -152,7 +146,7 @@ int mutex_spin_on_owner(struct mutex *lock, struct task_struct *owner)
                if (need_resched())
                        break;
 
-               arch_mutex_cpu_relax();
+               cpu_relax_lowlatency();
        }
        rcu_read_unlock();
 
@@ -388,12 +382,10 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
        /*
         * Optimistic spinning.
         *
-        * We try to spin for acquisition when we find that there are no
-        * pending waiters and the lock owner is currently running on a
-        * (different) CPU.
-        *
-        * The rationale is that if the lock owner is running, it is likely to
-        * release the lock soon.
+        * We try to spin for acquisition when we find that the lock owner
+        * is currently running on a (different) CPU and while we don't
+        * need to reschedule. The rationale is that if the lock owner is
+        * running, it is likely to release the lock soon.
         *
         * Since this needs the lock owner, and this mutex implementation
         * doesn't track the owner atomically in the lock field, we need to
@@ -440,7 +432,8 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
                if (owner && !mutex_spin_on_owner(lock, owner))
                        break;
 
-               if ((atomic_read(&lock->count) == 1) &&
+               /* Try to acquire the mutex if it is unlocked. */
+               if (!mutex_is_locked(lock) &&
                    (atomic_cmpxchg(&lock->count, 1, 0) == 1)) {
                        lock_acquired(&lock->dep_map, ip);
                        if (use_ww_ctx) {
@@ -471,7 +464,7 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
                 * memory barriers as we'll eventually observe the right
                 * values at the cost of a few extra spins.
                 */
-               arch_mutex_cpu_relax();
+               cpu_relax_lowlatency();
        }
        osq_unlock(&lock->osq);
 slowpath:
@@ -485,8 +478,11 @@ slowpath:
 #endif
        spin_lock_mutex(&lock->wait_lock, flags);
 
-       /* once more, can we acquire the lock? */
-       if (MUTEX_SHOW_NO_WAITER(lock) && (atomic_xchg(&lock->count, 0) == 1))
+       /*
+        * Once more, try to acquire the lock. Only try-lock the mutex if
+        * it is unlocked to reduce unnecessary xchg() operations.
+        */
+       if (!mutex_is_locked(lock) && (atomic_xchg(&lock->count, 0) == 1))
                goto skip_wait;
 
        debug_mutex_lock_common(lock, &waiter);
@@ -506,9 +502,10 @@ slowpath:
                 * it's unlocked. Later on, if we sleep, this is the
                 * operation that gives us the lock. We xchg it to -1, so
                 * that when we release the lock, we properly wake up the
-                * other waiters:
+                * other waiters. We only attempt the xchg if the count is
+                * non-negative in order to avoid unnecessary xchg operations:
                 */
-               if (MUTEX_SHOW_NO_WAITER(lock) &&
+               if (atomic_read(&lock->count) >= 0 &&
                    (atomic_xchg(&lock->count, -1) == 1))
                        break;
 
@@ -823,6 +820,10 @@ static inline int __mutex_trylock_slowpath(atomic_t *lock_count)
        unsigned long flags;
        int prev;
 
+       /* No need to trylock if the mutex is locked. */
+       if (mutex_is_locked(lock))
+               return 0;
+
        spin_lock_mutex(&lock->wait_lock, flags);
 
        prev = atomic_xchg(&lock->count, -1);