tcp: connect() race with timewait reuse

[firefly-linux-kernel-4.4.55.git] / net / ipv4 / inet_timewait_sock.c

diff --git a/net/ipv4/inet_timewait_sock.c b/net/ipv4/inet_timewait_sock.c
index 1f5d508bb18b80a4b8a3290ce0e3c0925b0e25f8..11a107a5af4f64d7a0e7159079941dc80359b2d4 100644 (file)
--- a/net/ipv4/inet_timewait_sock.c
+++ b/net/ipv4/inet_timewait_sock.c
@@ -14,22 +14,33 @@
 #include <net/inet_timewait_sock.h>
 #include <net/ip.h>
 
+
+/*
+ * unhash a timewait socket from established hash
+ * lock must be hold by caller
+ */
+int inet_twsk_unhash(struct inet_timewait_sock *tw)
+{
+       if (hlist_nulls_unhashed(&tw->tw_node))
+               return 0;
+
+       hlist_nulls_del_rcu(&tw->tw_node);
+       sk_nulls_node_init(&tw->tw_node);
+       return 1;
+}
+
 /* Must be called with locally disabled BHs. */
 static void __inet_twsk_kill(struct inet_timewait_sock *tw,
                             struct inet_hashinfo *hashinfo)
 {
        struct inet_bind_hashbucket *bhead;
        struct inet_bind_bucket *tb;
+       int refcnt;
        /* Unlink from established hashes. */
        spinlock_t *lock = inet_ehash_lockp(hashinfo, tw->tw_hash);
 
        spin_lock(lock);
-       if (hlist_nulls_unhashed(&tw->tw_node)) {
-               spin_unlock(lock);
-               return;
-       }
-       hlist_nulls_del_rcu(&tw->tw_node);
-       sk_nulls_node_init(&tw->tw_node);
+       refcnt = inet_twsk_unhash(tw);
        spin_unlock(lock);
 
        /* Disassociate with bind bucket. */
@@ -37,9 +48,12 @@ static void __inet_twsk_kill(struct inet_timewait_sock *tw,
                        hashinfo->bhash_size)];
        spin_lock(&bhead->lock);
        tb = tw->tw_tb;
-       __hlist_del(&tw->tw_bind_node);
-       tw->tw_tb = NULL;
-       inet_bind_bucket_destroy(hashinfo->bind_bucket_cachep, tb);
+       if (tb) {
+               __hlist_del(&tw->tw_bind_node);
+               tw->tw_tb = NULL;
+               inet_bind_bucket_destroy(hashinfo->bind_bucket_cachep, tb);
+               refcnt++;
+       }
        spin_unlock(&bhead->lock);
 #ifdef SOCK_REFCNT_DEBUG
        if (atomic_read(&tw->tw_refcnt) != 1) {
@@ -47,7 +61,10 @@ static void __inet_twsk_kill(struct inet_timewait_sock *tw,
                       tw->tw_prot->name, tw, atomic_read(&tw->tw_refcnt));
        }
 #endif
-       inet_twsk_put(tw);
+       while (refcnt) {
+               inet_twsk_put(tw);
+               refcnt--;
+       }
 }
 
 static noinline void inet_twsk_free(struct inet_timewait_sock *tw)
@@ -92,6 +109,7 @@ void __inet_twsk_hashdance(struct inet_timewait_sock *tw, struct sock *sk,
        tw->tw_tb = icsk->icsk_bind_hash;
        WARN_ON(!icsk->icsk_bind_hash);
        inet_twsk_add_bind_node(tw, &tw->tw_tb->owners);
+       atomic_inc(&tw->tw_refcnt);
        spin_unlock(&bhead->lock);
 
        spin_lock(lock);
@@ -421,37 +439,46 @@ out:
 
 EXPORT_SYMBOL_GPL(inet_twdr_twcal_tick);
 
-void inet_twsk_purge(struct net *net, struct inet_hashinfo *hashinfo,
+void inet_twsk_purge(struct inet_hashinfo *hashinfo,
                     struct inet_timewait_death_row *twdr, int family)
 {
        struct inet_timewait_sock *tw;
        struct sock *sk;
        struct hlist_nulls_node *node;
-       int h;
+       unsigned int slot;
 
-       local_bh_disable();
-       for (h = 0; h <= hashinfo->ehash_mask; h++) {
-               struct inet_ehash_bucket *head =
-                       inet_ehash_bucket(hashinfo, h);
-               spinlock_t *lock = inet_ehash_lockp(hashinfo, h);
+       for (slot = 0; slot <= hashinfo->ehash_mask; slot++) {
+               struct inet_ehash_bucket *head = &hashinfo->ehash[slot];
+restart_rcu:
+               rcu_read_lock();
 restart:
-               spin_lock(lock);
-               sk_nulls_for_each(sk, node, &head->twchain) {
-
+               sk_nulls_for_each_rcu(sk, node, &head->twchain) {
                        tw = inet_twsk(sk);
-                       if (!net_eq(twsk_net(tw), net) ||
-                           tw->tw_family != family)
+                       if ((tw->tw_family != family) ||
+                               atomic_read(&twsk_net(tw)->count))
                                continue;
 
-                       atomic_inc(&tw->tw_refcnt);
-                       spin_unlock(lock);
+                       if (unlikely(!atomic_inc_not_zero(&tw->tw_refcnt)))
+                               continue;
+
+                       if (unlikely((tw->tw_family != family) ||
+                                    atomic_read(&twsk_net(tw)->count))) {
+                               inet_twsk_put(tw);
+                               goto restart;
+                       }
+
+                       rcu_read_unlock();
                        inet_twsk_deschedule(tw, twdr);
                        inet_twsk_put(tw);
-
-                       goto restart;
+                       goto restart_rcu;
                }
-               spin_unlock(lock);
+               /* If the nulls value we got at the end of this lookup is
+                * not the expected one, we must restart lookup.
+                * We probably met an item that was moved to another chain.
+                */
+               if (get_nulls_value(node) != slot)
+                       goto restart;
+               rcu_read_unlock();
        }
-       local_bh_enable();
 }
 EXPORT_SYMBOL_GPL(inet_twsk_purge);