net-timestamp: SCHED timestamp on entering packet scheduler

Kernel transmit latency is often incurred in the packet scheduler.
Introduce a new timestamp on transmission just before entering the
scheduler. When data travels through multiple devices (bonding,
tunneling, ...) each device will export an individual timestamp.

Signed-off-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h
index 0e35b3af7317389b0093346f7f5334c0c4e86731..50e1e9b3a5a54ff3039c6cc02d070e56fa249d13 100644 (file)
--- a/include/linux/skbuff.h
+++ b/include/linux/skbuff.h
@@ -229,7 +229,7 @@ enum {
        /* generate hardware time stamp */
        SKBTX_HW_TSTAMP = 1 << 0,
 
-       /* generate software time stamp */
+       /* generate software time stamp when queueing packet to NIC */
        SKBTX_SW_TSTAMP = 1 << 1,
 
        /* device driver is going to provide hardware time stamp */
@@ -247,9 +247,12 @@ enum {
         * all frags to avoid possible bad checksum
         */
        SKBTX_SHARED_FRAG = 1 << 5,
+
+       /* generate software time stamp when entering packet scheduling */
+       SKBTX_SCHED_TSTAMP = 1 << 6,
 };
 
-#define SKBTX_ANY_SW_TSTAMP    SKBTX_SW_TSTAMP
+#define SKBTX_ANY_SW_TSTAMP    (SKBTX_SW_TSTAMP | SKBTX_SCHED_TSTAMP)
 #define SKBTX_ANY_TSTAMP       (SKBTX_HW_TSTAMP | SKBTX_ANY_SW_TSTAMP)
 
 /*
@@ -2695,6 +2698,10 @@ static inline bool skb_defer_rx_timestamp(struct sk_buff *skb)
 void skb_complete_tx_timestamp(struct sk_buff *skb,
                               struct skb_shared_hwtstamps *hwtstamps);
 
+void __skb_tstamp_tx(struct sk_buff *orig_skb,
+                    struct skb_shared_hwtstamps *hwtstamps,
+                    struct sock *sk, int tstype);
+
 /**
  * skb_tstamp_tx - queue clone of skb with send time stamps
  * @orig_skb:  the original outgoing packet

diff --git a/include/uapi/linux/errqueue.h b/include/uapi/linux/errqueue.h
index accee72cae7cc640ada78ac9f6d95aeb4d57e88f..17437cf297b714f034dd1265beae9dfc196a67c8 100644 (file)
--- a/include/uapi/linux/errqueue.h
+++ b/include/uapi/linux/errqueue.h
@@ -39,6 +39,7 @@ struct scm_timestamping {
  */
 enum {
        SCM_TSTAMP_SND,         /* driver passed skb to NIC, or HW */
+       SCM_TSTAMP_SCHED,       /* data entered the packet scheduler */
 };
 
 #endif /* _UAPI_LINUX_ERRQUEUE_H */

diff --git a/include/uapi/linux/net_tstamp.h b/include/uapi/linux/net_tstamp.h
index 1e861d2e1a315df6829db23971df02c9e0952bb3..60733845fcdd0ab835a4ea7872515792ede703f6 100644 (file)
--- a/include/uapi/linux/net_tstamp.h
+++ b/include/uapi/linux/net_tstamp.h
@@ -21,8 +21,9 @@ enum {
        SOF_TIMESTAMPING_SYS_HARDWARE = (1<<5),
        SOF_TIMESTAMPING_RAW_HARDWARE = (1<<6),
        SOF_TIMESTAMPING_OPT_ID = (1<<7),
+       SOF_TIMESTAMPING_TX_SCHED = (1<<8),
 
-       SOF_TIMESTAMPING_LAST = SOF_TIMESTAMPING_OPT_ID,
+       SOF_TIMESTAMPING_LAST = SOF_TIMESTAMPING_TX_SCHED,
        SOF_TIMESTAMPING_MASK = (SOF_TIMESTAMPING_LAST - 1) |
                                 SOF_TIMESTAMPING_LAST
 };

diff --git a/net/core/dev.c b/net/core/dev.c
index b370230fe1d3f27ee1f77c32f079d5e28da09b3c..1c15b189c52b24e5f2d63172299e555387c2b3d9 100644 (file)
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -132,6 +132,7 @@
 #include <linux/hashtable.h>
 #include <linux/vmalloc.h>
 #include <linux/if_macvlan.h>
+#include <linux/errqueue.h>
 
 #include "net-sysfs.h"
 
@@ -2876,6 +2877,9 @@ static int __dev_queue_xmit(struct sk_buff *skb, void *accel_priv)
 
        skb_reset_mac_header(skb);
 
+       if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP))
+               __skb_tstamp_tx(skb, NULL, skb->sk, SCM_TSTAMP_SCHED);
+
        /* Disable soft irqs for various locks below. Also
         * stops preemption for RCU.
         */

diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index 0df4f1d14c5a98a104b19958f1449de8c3090739..9705c0732aabeced7c84f7335c83596074704d19 100644 (file)
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -3490,10 +3490,10 @@ int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb)
 }
 EXPORT_SYMBOL(sock_queue_err_skb);
 
-void skb_tstamp_tx(struct sk_buff *orig_skb,
-               struct skb_shared_hwtstamps *hwtstamps)
+void __skb_tstamp_tx(struct sk_buff *orig_skb,
+                    struct skb_shared_hwtstamps *hwtstamps,
+                    struct sock *sk, int tstype)
 {
-       struct sock *sk = orig_skb->sk;
        struct sock_exterr_skb *serr;
        struct sk_buff *skb;
        int err;
@@ -3521,7 +3521,7 @@ void skb_tstamp_tx(struct sk_buff *orig_skb,
        memset(serr, 0, sizeof(*serr));
        serr->ee.ee_errno = ENOMSG;
        serr->ee.ee_origin = SO_EE_ORIGIN_TIMESTAMPING;
-       serr->ee.ee_info = SCM_TSTAMP_SND;
+       serr->ee.ee_info = tstype;
        if (sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID)
                serr->ee.ee_data = skb_shinfo(skb)->tskey;
 
@@ -3530,6 +3530,14 @@ void skb_tstamp_tx(struct sk_buff *orig_skb,
        if (err)
                kfree_skb(skb);
 }
+EXPORT_SYMBOL_GPL(__skb_tstamp_tx);
+
+void skb_tstamp_tx(struct sk_buff *orig_skb,
+                  struct skb_shared_hwtstamps *hwtstamps)
+{
+       return __skb_tstamp_tx(orig_skb, hwtstamps, orig_skb->sk,
+                              SCM_TSTAMP_SND);
+}
 EXPORT_SYMBOL_GPL(skb_tstamp_tx);
 
 void skb_complete_wifi_ack(struct sk_buff *skb, bool acked)

diff --git a/net/socket.c b/net/socket.c
index 255d9b80272388d8896e2b740b5942eed60ae31b..3a2778d71631d4991cee7016e661e76901b41ab2 100644 (file)
--- a/net/socket.c
+++ b/net/socket.c
@@ -617,6 +617,9 @@ void sock_tx_timestamp(struct sock *sk, __u8 *tx_flags)
                *tx_flags |= SKBTX_HW_TSTAMP;
        if (sk->sk_tsflags & SOF_TIMESTAMPING_TX_SOFTWARE)
                *tx_flags |= SKBTX_SW_TSTAMP;
+       if (sk->sk_tsflags & SOF_TIMESTAMPING_TX_SCHED)
+               *tx_flags |= SKBTX_SCHED_TSTAMP;
+
        if (sock_flag(sk, SOCK_WIFI_STATUS))
                *tx_flags |= SKBTX_WIFI_STATUS;
 }