netfilter: xtables: use percpu rule counters

[firefly-linux-kernel-4.4.55.git] / net / ipv4 / inet_hashtables.c

diff --git a/net/ipv4/inet_hashtables.c b/net/ipv4/inet_hashtables.c
index c6fb80bd5826ea840eebd033fb87d01c595ab120..5f9b063bbe8ab4f3755a5711ae19b816a3bc2026 100644 (file)
--- a/net/ipv4/inet_hashtables.c
+++ b/net/ipv4/inet_hashtables.c
@@ -18,6 +18,7 @@
 #include <linux/sched.h>
 #include <linux/slab.h>
 #include <linux/wait.h>
+#include <linux/vmalloc.h>
 
 #include <net/inet_connection_sock.h>
 #include <net/inet_hashtables.h>
@@ -90,10 +91,6 @@ void inet_bind_bucket_destroy(struct kmem_cache *cachep, struct inet_bind_bucket
 void inet_bind_hash(struct sock *sk, struct inet_bind_bucket *tb,
                    const unsigned short snum)
 {
-       struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
-
-       atomic_inc(&hashinfo->bsockets);
-
        inet_sk(sk)->inet_num = snum;
        sk_add_bind_node(sk, &tb->owners);
        tb->num_owners++;
@@ -111,8 +108,6 @@ static void __inet_put_port(struct sock *sk)
        struct inet_bind_hashbucket *head = &hashinfo->bhash[bhash];
        struct inet_bind_bucket *tb;
 
-       atomic_dec(&hashinfo->bsockets);
-
        spin_lock(&head->lock);
        tb = inet_csk(sk)->icsk_bind_hash;
        __sk_del_bind_node(sk);
@@ -399,9 +394,10 @@ not_unique:
        return -EADDRNOTAVAIL;
 }
 
-static inline u32 inet_sk_port_offset(const struct sock *sk)
+static u32 inet_sk_port_offset(const struct sock *sk)
 {
        const struct inet_sock *inet = inet_sk(sk);
+
        return secure_ipv4_port_ephemeral(inet->inet_rcv_saddr,
                                          inet->inet_daddr,
                                          inet->inet_dport);
@@ -507,8 +503,14 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row,
                inet_get_local_port_range(net, &low, &high);
                remaining = (high - low) + 1;
 
+               /* By starting with offset being an even number,
+                * we tend to leave about 50% of ports for other uses,
+                * like bind(0).
+                */
+               offset &= ~1;
+
                local_bh_disable();
-               for (i = 1; i <= remaining; i++) {
+               for (i = 0; i < remaining; i++) {
                        port = low + (i + offset) % remaining;
                        if (inet_is_local_reserved_port(net, port))
                                continue;
@@ -552,7 +554,7 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row,
                return -EADDRNOTAVAIL;
 
 ok:
-               hint += i;
+               hint += (i + 2) & ~1;
 
                /* Head lock still held and bh's disabled */
                inet_bind_hash(sk, tb, port);
@@ -599,7 +601,11 @@ out:
 int inet_hash_connect(struct inet_timewait_death_row *death_row,
                      struct sock *sk)
 {
-       return __inet_hash_connect(death_row, sk, inet_sk_port_offset(sk),
+       u32 port_offset = 0;
+
+       if (!inet_sk(sk)->inet_num)
+               port_offset = inet_sk_port_offset(sk);
+       return __inet_hash_connect(death_row, sk, port_offset,
                                   __inet_check_established);
 }
 EXPORT_SYMBOL_GPL(inet_hash_connect);
@@ -608,7 +614,6 @@ void inet_hashinfo_init(struct inet_hashinfo *h)
 {
        int i;
 
-       atomic_set(&h->bsockets, 0);
        for (i = 0; i < INET_LHTABLE_SIZE; i++) {
                spin_lock_init(&h->listening_hash[i].lock);
                INIT_HLIST_NULLS_HEAD(&h->listening_hash[i].head,
@@ -616,3 +621,33 @@ void inet_hashinfo_init(struct inet_hashinfo *h)
                }
 }
 EXPORT_SYMBOL_GPL(inet_hashinfo_init);
+
+int inet_ehash_locks_alloc(struct inet_hashinfo *hashinfo)
+{
+       unsigned int i, nblocks = 1;
+
+       if (sizeof(spinlock_t) != 0) {
+               /* allocate 2 cache lines or at least one spinlock per cpu */
+               nblocks = max_t(unsigned int,
+                               2 * L1_CACHE_BYTES / sizeof(spinlock_t),
+                               1);
+               nblocks = roundup_pow_of_two(nblocks * num_possible_cpus());
+
+               /* no more locks than number of hash buckets */
+               nblocks = min(nblocks, hashinfo->ehash_mask + 1);
+
+               hashinfo->ehash_locks = kmalloc_array(nblocks, sizeof(spinlock_t),
+                                                     GFP_KERNEL | __GFP_NOWARN);
+               if (!hashinfo->ehash_locks)
+                       hashinfo->ehash_locks = vmalloc(nblocks * sizeof(spinlock_t));
+
+               if (!hashinfo->ehash_locks)
+                       return -ENOMEM;
+
+               for (i = 0; i < nblocks; i++)
+                       spin_lock_init(&hashinfo->ehash_locks[i]);
+       }
+       hashinfo->ehash_locks_mask = nblocks - 1;
+       return 0;
+}
+EXPORT_SYMBOL_GPL(inet_ehash_locks_alloc);