net_sched: add 64bit rate estimators

struct gnet_stats_rate_est contains u32 fields, so the bytes per second
field can wrap at 34360Mbit.

Add a new gnet_stats_rate_est64 structure to get 64bit bps/pps fields,
and switch the kernel to use this structure natively.

This structure is dumped to user space as a new attribute :

TCA_STATS_RATE_EST64

Old tc command will now display the capped bps (to 34360Mbit), instead
of wrapped values, and updated tc command will display correct
information.

Old tc command output, after patch :

eric:~# tc -s -d qd sh dev lo
qdisc pfifo 8001: root refcnt 2 limit 1000p
 Sent 80868245400 bytes 1978837 pkt (dropped 0, overlimits 0 requeues 0)
 rate 34360Mbit 189696pps backlog 0b 0p requeues 0

This patch carefully reorganizes "struct Qdisc" layout to get optimal
performance on SMP.

Signed-off-by: Eric Dumazet <edumazet@google.com>
Cc: Ben Hutchings <bhutchings@solarflare.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/include/net/act_api.h b/include/net/act_api.h
index 06ef7e926a66483633763a6079bd3ab5d8c40170..b8ffac7b6bab755f5adb4d648b5d34bd57eb028f 100644 (file)
--- a/include/net/act_api.h
+++ b/include/net/act_api.h
@@ -18,7 +18,7 @@ struct tcf_common {
        struct tcf_t                    tcfc_tm;
        struct gnet_stats_basic_packed  tcfc_bstats;
        struct gnet_stats_queue         tcfc_qstats;
-       struct gnet_stats_rate_est      tcfc_rate_est;
+       struct gnet_stats_rate_est64    tcfc_rate_est;
        spinlock_t                      tcfc_lock;
        struct rcu_head                 tcfc_rcu;
 };

diff --git a/include/net/gen_stats.h b/include/net/gen_stats.h
index a79b6cfb02a8dcc252afa516e13f76f91af6eb67..cf8439ba4d11062a326919fa885c3ae5a24e62c5 100644 (file)
--- a/include/net/gen_stats.h
+++ b/include/net/gen_stats.h
@@ -30,7 +30,7 @@ extern int gnet_stats_copy_basic(struct gnet_dump *d,
                                 struct gnet_stats_basic_packed *b);
 extern int gnet_stats_copy_rate_est(struct gnet_dump *d,
                                    const struct gnet_stats_basic_packed *b,
-                                   struct gnet_stats_rate_est *r);
+                                   struct gnet_stats_rate_est64 *r);
 extern int gnet_stats_copy_queue(struct gnet_dump *d,
                                 struct gnet_stats_queue *q);
 extern int gnet_stats_copy_app(struct gnet_dump *d, void *st, int len);
@@ -38,13 +38,13 @@ extern int gnet_stats_copy_app(struct gnet_dump *d, void *st, int len);
 extern int gnet_stats_finish_copy(struct gnet_dump *d);
 
 extern int gen_new_estimator(struct gnet_stats_basic_packed *bstats,
-                            struct gnet_stats_rate_est *rate_est,
+                            struct gnet_stats_rate_est64 *rate_est,
                             spinlock_t *stats_lock, struct nlattr *opt);
 extern void gen_kill_estimator(struct gnet_stats_basic_packed *bstats,
-                              struct gnet_stats_rate_est *rate_est);
+                              struct gnet_stats_rate_est64 *rate_est);
 extern int gen_replace_estimator(struct gnet_stats_basic_packed *bstats,
-                                struct gnet_stats_rate_est *rate_est,
+                                struct gnet_stats_rate_est64 *rate_est,
                                 spinlock_t *stats_lock, struct nlattr *opt);
 extern bool gen_estimator_active(const struct gnet_stats_basic_packed *bstats,
-                                const struct gnet_stats_rate_est *rate_est);
+                                const struct gnet_stats_rate_est64 *rate_est);
 #endif

diff --git a/include/net/netfilter/xt_rateest.h b/include/net/netfilter/xt_rateest.h
index 5a2978d1cb22ae8713d43688b74370e44727725e..495c71f66e7e9334b3506174dc7e5352399938f4 100644 (file)
--- a/include/net/netfilter/xt_rateest.h
+++ b/include/net/netfilter/xt_rateest.h
@@ -6,7 +6,7 @@ struct xt_rateest {
        struct gnet_stats_basic_packed  bstats;
        spinlock_t                      lock;
        /* keep rstats and lock on same cache line to speedup xt_rateest_mt() */
-       struct gnet_stats_rate_est      rstats;
+       struct gnet_stats_rate_est64    rstats;
 
        /* following fields not accessed in hot path */
        struct hlist_node               list;

diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h
index e7f4e21cc3e17ec53ac8c5021daa74401766decb..df5676029827dc021b4c0a1ddc30e60bdc5a6447 100644 (file)
--- a/include/net/sch_generic.h
+++ b/include/net/sch_generic.h
@@ -58,14 +58,12 @@ struct Qdisc {
                                      * multiqueue device.
                                      */
 #define TCQ_F_WARN_NONWC       (1 << 16)
-       int                     padded;
+       u32                     limit;
        const struct Qdisc_ops  *ops;
        struct qdisc_size_table __rcu *stab;
        struct list_head        list;
        u32                     handle;
        u32                     parent;
-       atomic_t                refcnt;
-       struct gnet_stats_rate_est      rate_est;
        int                     (*reshape_fail)(struct sk_buff *skb,
                                        struct Qdisc *q);
 
@@ -76,8 +74,9 @@ struct Qdisc {
         */
        struct Qdisc            *__parent;
        struct netdev_queue     *dev_queue;
-       struct Qdisc            *next_sched;
 
+       struct gnet_stats_rate_est64    rate_est;
+       struct Qdisc            *next_sched;
        struct sk_buff          *gso_skb;
        /*
         * For performance sake on SMP, we put highly modified fields at the end
@@ -88,8 +87,10 @@ struct Qdisc {
        unsigned int            __state;
        struct gnet_stats_queue qstats;
        struct rcu_head         rcu_head;
-       spinlock_t              busylock;
-       u32                     limit;
+       int                     padded;
+       atomic_t                refcnt;
+
+       spinlock_t              busylock ____cacheline_aligned_in_smp;
 };
 
 static inline bool qdisc_is_running(const struct Qdisc *qdisc)

diff --git a/include/uapi/linux/gen_stats.h b/include/uapi/linux/gen_stats.h
index 552c8a0a12d15f11f6ac2b3b1a27d17a94044f8b..6487317ea619c41d4ea662f1162417fa6c6e947f 100644 (file)
--- a/include/uapi/linux/gen_stats.h
+++ b/include/uapi/linux/gen_stats.h
@@ -9,6 +9,7 @@ enum {
        TCA_STATS_RATE_EST,
        TCA_STATS_QUEUE,
        TCA_STATS_APP,
+       TCA_STATS_RATE_EST64,
        __TCA_STATS_MAX,
 };
 #define TCA_STATS_MAX (__TCA_STATS_MAX - 1)
@@ -37,6 +38,16 @@ struct gnet_stats_rate_est {
        __u32   pps;
 };
 
+/**
+ * struct gnet_stats_rate_est64 - rate estimator
+ * @bps: current byte rate
+ * @pps: current packet rate
+ */
+struct gnet_stats_rate_est64 {
+       __u64   bps;
+       __u64   pps;
+};
+
 /**
  * struct gnet_stats_queue - queuing statistics
  * @qlen: queue length

diff --git a/net/core/gen_estimator.c b/net/core/gen_estimator.c
index d9d198aa9fed0738288973815b427ec6bae0de4a..6b5b6e7013cafec8a7b3767ed97a610bf35f1ae6 100644 (file)
--- a/net/core/gen_estimator.c
+++ b/net/core/gen_estimator.c
@@ -82,7 +82,7 @@ struct gen_estimator
 {
        struct list_head        list;
        struct gnet_stats_basic_packed  *bstats;
-       struct gnet_stats_rate_est      *rate_est;
+       struct gnet_stats_rate_est64    *rate_est;
        spinlock_t              *stats_lock;
        int                     ewma_log;
        u64                     last_bytes;
@@ -167,7 +167,7 @@ static void gen_add_node(struct gen_estimator *est)
 
 static
 struct gen_estimator *gen_find_node(const struct gnet_stats_basic_packed *bstats,
-                                   const struct gnet_stats_rate_est *rate_est)
+                                   const struct gnet_stats_rate_est64 *rate_est)
 {
        struct rb_node *p = est_root.rb_node;
 
@@ -203,7 +203,7 @@ struct gen_estimator *gen_find_node(const struct gnet_stats_basic_packed *bstats
  *
  */
 int gen_new_estimator(struct gnet_stats_basic_packed *bstats,
-                     struct gnet_stats_rate_est *rate_est,
+                     struct gnet_stats_rate_est64 *rate_est,
                      spinlock_t *stats_lock,
                      struct nlattr *opt)
 {
@@ -258,7 +258,7 @@ EXPORT_SYMBOL(gen_new_estimator);
  * Note : Caller should respect an RCU grace period before freeing stats_lock
  */
 void gen_kill_estimator(struct gnet_stats_basic_packed *bstats,
-                       struct gnet_stats_rate_est *rate_est)
+                       struct gnet_stats_rate_est64 *rate_est)
 {
        struct gen_estimator *e;
 
@@ -290,7 +290,7 @@ EXPORT_SYMBOL(gen_kill_estimator);
  * Returns 0 on success or a negative error code.
  */
 int gen_replace_estimator(struct gnet_stats_basic_packed *bstats,
-                         struct gnet_stats_rate_est *rate_est,
+                         struct gnet_stats_rate_est64 *rate_est,
                          spinlock_t *stats_lock, struct nlattr *opt)
 {
        gen_kill_estimator(bstats, rate_est);
@@ -306,7 +306,7 @@ EXPORT_SYMBOL(gen_replace_estimator);
  * Returns true if estimator is active, and false if not.
  */
 bool gen_estimator_active(const struct gnet_stats_basic_packed *bstats,
-                         const struct gnet_stats_rate_est *rate_est)
+                         const struct gnet_stats_rate_est64 *rate_est)
 {
        bool res;
 

diff --git a/net/core/gen_stats.c b/net/core/gen_stats.c
index ddedf211e588146f17eb2e147307d04991720170..9d3d9e78397b0f90f379addab25c7fd78294b3bc 100644 (file)
--- a/net/core/gen_stats.c
+++ b/net/core/gen_stats.c
@@ -143,18 +143,30 @@ EXPORT_SYMBOL(gnet_stats_copy_basic);
 int
 gnet_stats_copy_rate_est(struct gnet_dump *d,
                         const struct gnet_stats_basic_packed *b,
-                        struct gnet_stats_rate_est *r)
+                        struct gnet_stats_rate_est64 *r)
 {
+       struct gnet_stats_rate_est est;
+       int res;
+
        if (b && !gen_estimator_active(b, r))
                return 0;
 
+       est.bps = min_t(u64, UINT_MAX, r->bps);
+       /* we have some time before reaching 2^32 packets per second */
+       est.pps = r->pps;
+
        if (d->compat_tc_stats) {
-               d->tc_stats.bps = r->bps;
-               d->tc_stats.pps = r->pps;
+               d->tc_stats.bps = est.bps;
+               d->tc_stats.pps = est.pps;
        }
 
-       if (d->tail)
-               return gnet_stats_copy(d, TCA_STATS_RATE_EST, r, sizeof(*r));
+       if (d->tail) {
+               res = gnet_stats_copy(d, TCA_STATS_RATE_EST, &est, sizeof(est));
+               if (res < 0 || est.bps == r->bps)
+                       return res;
+               /* emit 64bit stats only if needed */
+               return gnet_stats_copy(d, TCA_STATS_RATE_EST64, r, sizeof(*r));
+       }
 
        return 0;
 }

diff --git a/net/netfilter/xt_rateest.c b/net/netfilter/xt_rateest.c
index ed0db15ab00e3c4f07f5931e27bc1f657988934c..7720b036d76a84c949080c8c32827b604c80da64 100644 (file)
--- a/net/netfilter/xt_rateest.c
+++ b/net/netfilter/xt_rateest.c
@@ -18,7 +18,7 @@ static bool
 xt_rateest_mt(const struct sk_buff *skb, struct xt_action_param *par)
 {
        const struct xt_rateest_match_info *info = par->matchinfo;
-       struct gnet_stats_rate_est *r;
+       struct gnet_stats_rate_est64 *r;
        u_int32_t bps1, bps2, pps1, pps2;
        bool ret = true;
 

diff --git a/net/sched/sch_cbq.c b/net/sched/sch_cbq.c
index 1bc210ffcba2a524750b3382d444a87db2c08c2c..71a568862557c26cb9fd97bfbd26531b55ad4283 100644 (file)
--- a/net/sched/sch_cbq.c
+++ b/net/sched/sch_cbq.c
@@ -130,7 +130,7 @@ struct cbq_class {
        psched_time_t           penalized;
        struct gnet_stats_basic_packed bstats;
        struct gnet_stats_queue qstats;
-       struct gnet_stats_rate_est rate_est;
+       struct gnet_stats_rate_est64 rate_est;
        struct tc_cbq_xstats    xstats;
 
        struct tcf_proto        *filter_list;

diff --git a/net/sched/sch_drr.c b/net/sched/sch_drr.c
index 759b308d1a8d2ac427893b1258d4ff508e963963..8302717ea3034247bc27597e067ac8771dc973b5 100644 (file)
--- a/net/sched/sch_drr.c
+++ b/net/sched/sch_drr.c
@@ -25,7 +25,7 @@ struct drr_class {
 
        struct gnet_stats_basic_packed          bstats;
        struct gnet_stats_queue         qstats;
-       struct gnet_stats_rate_est      rate_est;
+       struct gnet_stats_rate_est64    rate_est;
        struct list_head                alist;
        struct Qdisc                    *qdisc;
 

diff --git a/net/sched/sch_hfsc.c b/net/sched/sch_hfsc.c
index 9facea03faeb89e6b51f54a2f4c01729979b8584..c4075610502cf3f53f4ac3f1bc1be7b096e7f488 100644 (file)
--- a/net/sched/sch_hfsc.c
+++ b/net/sched/sch_hfsc.c
@@ -114,7 +114,7 @@ struct hfsc_class {
 
        struct gnet_stats_basic_packed bstats;
        struct gnet_stats_queue qstats;
-       struct gnet_stats_rate_est rate_est;
+       struct gnet_stats_rate_est64 rate_est;
        unsigned int    level;          /* class level in hierarchy */
        struct tcf_proto *filter_list;  /* filter list */
        unsigned int    filter_cnt;     /* filter count */

diff --git a/net/sched/sch_htb.c b/net/sched/sch_htb.c
index adaedd79389ccd0775a824c9e3c61aef2ea5355e..162fb800754cc4660b50c272e1c288f44e82faeb 100644 (file)
--- a/net/sched/sch_htb.c
+++ b/net/sched/sch_htb.c
@@ -78,7 +78,7 @@ struct htb_class {
        /* general class parameters */
        struct gnet_stats_basic_packed bstats;
        struct gnet_stats_queue qstats;
-       struct gnet_stats_rate_est rate_est;
+       struct gnet_stats_rate_est64 rate_est;
        struct tc_htb_xstats xstats;    /* our special stats */
        int refcnt;             /* usage count of this class */
 

diff --git a/net/sched/sch_qfq.c b/net/sched/sch_qfq.c
index d51852bba01c981c9f9834dad82cfbcfec904508..7c195d972bf0cd437a8b94a6bc80564c1585b82b 100644 (file)
--- a/net/sched/sch_qfq.c
+++ b/net/sched/sch_qfq.c
@@ -138,7 +138,7 @@ struct qfq_class {
 
        struct gnet_stats_basic_packed bstats;
        struct gnet_stats_queue qstats;
-       struct gnet_stats_rate_est rate_est;
+       struct gnet_stats_rate_est64 rate_est;
        struct Qdisc *qdisc;
        struct list_head alist;         /* Link for active-classes list. */
        struct qfq_aggregate *agg;      /* Parent aggregate. */