pkt_sched: Add multiqueue scheduler support
authorAlexander Duyck <alexander.h.duyck@intel.com>
Fri, 12 Sep 2008 23:29:34 +0000 (16:29 -0700)
committerDavid S. Miller <davem@davemloft.net>
Fri, 12 Sep 2008 23:29:34 +0000 (16:29 -0700)
This patch is intended to add a qdisc to support the new tx multiqueue
architecture by providing a band for each hardware queue.  By doing
this it is possible to support a different qdisc per physical hardware
queue.

This qdisc uses the skb->queue_mapping to select which band to place
the traffic onto.  It then uses a round robin w/ a check to see if the
subqueue is stopped to determine which band to dequeue the packet from.

Signed-off-by: Alexander Duyck <alexander.h.duyck@intel.com>
Signed-off-by: Jeff Kirsher <jeffrey.t.kirsher@intel.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
Documentation/networking/multiqueue.txt
include/linux/pkt_sched.h
net/sched/Kconfig
net/sched/Makefile
net/sched/sch_multiq.c [new file with mode: 0644]

index d391ea63114132a85c4b922e1eca1e6960cfc7af..5787ee6eca4f1deb5eea8f1745bbe0ee9ca3e34a 100644 (file)
@@ -24,4 +24,49 @@ netif_{start|stop|wake}_subqueue() functions to manage each queue while the
 device is still operational.  netdev->queue_lock is still used when the device
 comes online or when it's completely shut down (unregister_netdev(), etc.).
 
-Author: Peter P. Waskiewicz Jr. <peter.p.waskiewicz.jr@intel.com>
+
+Section 2: Qdisc support for multiqueue devices
+
+-----------------------------------------------
+
+Currently two qdiscs support multiqueue devices.  The first is the default
+pfifo_fast qdisc.  This qdisc supports one qdisc per hardware queue.  A new
+round-robin qdisc, sch_multiq also supports multiple hardware queues. The
+qdisc is responsible for classifying the skb's and then directing the skb's to
+bands and queues based on the value in skb->queue_mapping.  Use this field in
+the base driver to determine which queue to send the skb to.
+
+sch_multiq has been added for hardware that wishes to avoid unnecessary
+requeuing.  It will cycle though the bands and verify that the hardware queue
+associated with the band is not stopped prior to dequeuing a packet.
+
+On qdisc load, the number of bands is based on the number of queues on the
+hardware.  Once the association is made, any skb with skb->queue_mapping set,
+will be queued to the band associated with the hardware queue.
+
+
+Section 3: Brief howto using MULTIQ for multiqueue devices
+---------------------------------------------------------------
+
+The userspace command 'tc,' part of the iproute2 package, is used to configure
+qdiscs.  To add the MULTIQ qdisc to your network device, assuming the device
+is called eth0, run the following command:
+
+# tc qdisc add dev eth0 root handle 1: multiq
+
+The qdisc will allocate the number of bands to equal the number of queues that
+the device reports, and bring the qdisc online.  Assuming eth0 has 4 Tx
+queues, the band mapping would look like:
+
+band 0 => queue 0
+band 1 => queue 1
+band 2 => queue 2
+band 3 => queue 3
+
+Traffic will begin flowing through each queue if your base device has either
+the default simple_tx_hash or a custom netdev->select_queue() defined.
+
+The behavior of tc filters remains the same.
+
+Author: Alexander Duyck <alexander.h.duyck@intel.com>
+Original Author: Peter P. Waskiewicz Jr. <peter.p.waskiewicz.jr@intel.com>
index e5de421ac7b422dc2526837512183311446b0884..5d921fa91a5bdb9e59ac2eec0036ed58082c9475 100644 (file)
@@ -123,6 +123,13 @@ struct tc_prio_qopt
        __u8    priomap[TC_PRIO_MAX+1]; /* Map: logical priority -> PRIO band */
 };
 
+/* MULTIQ section */
+
+struct tc_multiq_qopt {
+       __u16   bands;                  /* Number of bands */
+       __u16   max_bands;              /* Maximum number of queues */
+};
+
 /* TBF section */
 
 struct tc_tbf_qopt
index 9437b27ff84de101b88b3cbe08675f002e562238..efaa7a75e7f3782a1ba6d63ec34a039e4bc626f9 100644 (file)
@@ -106,6 +106,15 @@ config NET_SCH_PRIO
          To compile this code as a module, choose M here: the
          module will be called sch_prio.
 
+config NET_SCH_MULTIQ
+       tristate "Hardware Multiqueue-aware Multi Band Queuing (MULTIQ)"
+       ---help---
+         Say Y here if you want to use an n-band queue packet scheduler
+         to support devices that have multiple hardware transmit queues.
+
+         To compile this code as a module, choose M here: the
+         module will be called sch_multiq.
+
 config NET_SCH_RED
        tristate "Random Early Detection (RED)"
        ---help---
index 1d2b0f7df8487bbfbb633230c7dffe998f8257eb..3d9b953f7f6293d7e2e23ed1e3127e76e192f78d 100644 (file)
@@ -26,6 +26,7 @@ obj-$(CONFIG_NET_SCH_SFQ)     += sch_sfq.o
 obj-$(CONFIG_NET_SCH_TBF)      += sch_tbf.o
 obj-$(CONFIG_NET_SCH_TEQL)     += sch_teql.o
 obj-$(CONFIG_NET_SCH_PRIO)     += sch_prio.o
+obj-$(CONFIG_NET_SCH_MULTIQ)   += sch_multiq.o
 obj-$(CONFIG_NET_SCH_ATM)      += sch_atm.o
 obj-$(CONFIG_NET_SCH_NETEM)    += sch_netem.o
 obj-$(CONFIG_NET_CLS_U32)      += cls_u32.o
diff --git a/net/sched/sch_multiq.c b/net/sched/sch_multiq.c
new file mode 100644 (file)
index 0000000..49a8b67
--- /dev/null
@@ -0,0 +1,467 @@
+/*
+ * Copyright (c) 2008, Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+ * Place - Suite 330, Boston, MA 02111-1307 USA.
+ *
+ * Author: Alexander Duyck <alexander.h.duyck@intel.com>
+ */
+
+#include <linux/module.h>
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/string.h>
+#include <linux/errno.h>
+#include <linux/skbuff.h>
+#include <net/netlink.h>
+#include <net/pkt_sched.h>
+
+
+struct multiq_sched_data {
+       u16 bands;
+       u16 max_bands;
+       u16 curband;
+       struct tcf_proto *filter_list;
+       struct Qdisc **queues;
+};
+
+
+static struct Qdisc *
+multiq_classify(struct sk_buff *skb, struct Qdisc *sch, int *qerr)
+{
+       struct multiq_sched_data *q = qdisc_priv(sch);
+       u32 band;
+       struct tcf_result res;
+       int err;
+
+       *qerr = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS;
+       err = tc_classify(skb, q->filter_list, &res);
+#ifdef CONFIG_NET_CLS_ACT
+       switch (err) {
+       case TC_ACT_STOLEN:
+       case TC_ACT_QUEUED:
+               *qerr = NET_XMIT_SUCCESS | __NET_XMIT_STOLEN;
+       case TC_ACT_SHOT:
+               return NULL;
+       }
+#endif
+       band = skb_get_queue_mapping(skb);
+
+       if (band >= q->bands)
+               return q->queues[0];
+
+       return q->queues[band];
+}
+
+static int
+multiq_enqueue(struct sk_buff *skb, struct Qdisc *sch)
+{
+       struct Qdisc *qdisc;
+       int ret;
+
+       qdisc = multiq_classify(skb, sch, &ret);
+#ifdef CONFIG_NET_CLS_ACT
+       if (qdisc == NULL) {
+
+               if (ret & __NET_XMIT_BYPASS)
+                       sch->qstats.drops++;
+               kfree_skb(skb);
+               return ret;
+       }
+#endif
+
+       ret = qdisc_enqueue(skb, qdisc);
+       if (ret == NET_XMIT_SUCCESS) {
+               sch->bstats.bytes += qdisc_pkt_len(skb);
+               sch->bstats.packets++;
+               sch->q.qlen++;
+               return NET_XMIT_SUCCESS;
+       }
+       if (net_xmit_drop_count(ret))
+               sch->qstats.drops++;
+       return ret;
+}
+
+
+static int
+multiq_requeue(struct sk_buff *skb, struct Qdisc *sch)
+{
+       struct Qdisc *qdisc;
+       int ret;
+
+       qdisc = multiq_classify(skb, sch, &ret);
+#ifdef CONFIG_NET_CLS_ACT
+       if (qdisc == NULL) {
+               if (ret & __NET_XMIT_BYPASS)
+                       sch->qstats.drops++;
+               kfree_skb(skb);
+               return ret;
+       }
+#endif
+
+       ret = qdisc->ops->requeue(skb, qdisc);
+       if (ret == NET_XMIT_SUCCESS) {
+               sch->q.qlen++;
+               sch->qstats.requeues++;
+               return NET_XMIT_SUCCESS;
+       }
+       if (net_xmit_drop_count(ret))
+               sch->qstats.drops++;
+       return ret;
+}
+
+
+static struct sk_buff *multiq_dequeue(struct Qdisc *sch)
+{
+       struct multiq_sched_data *q = qdisc_priv(sch);
+       struct Qdisc *qdisc;
+       struct sk_buff *skb;
+       int band;
+
+       for (band = 0; band < q->bands; band++) {
+               /* cycle through bands to ensure fairness */
+               q->curband++;
+               if (q->curband >= q->bands)
+                       q->curband = 0;
+
+               /* Check that target subqueue is available before
+                * pulling an skb to avoid excessive requeues
+                */
+               if (!__netif_subqueue_stopped(qdisc_dev(sch), q->curband)) {
+                       qdisc = q->queues[q->curband];
+                       skb = qdisc->dequeue(qdisc);
+                       if (skb) {
+                               sch->q.qlen--;
+                               return skb;
+                       }
+               }
+       }
+       return NULL;
+
+}
+
+static unsigned int multiq_drop(struct Qdisc *sch)
+{
+       struct multiq_sched_data *q = qdisc_priv(sch);
+       int band;
+       unsigned int len;
+       struct Qdisc *qdisc;
+
+       for (band = q->bands-1; band >= 0; band--) {
+               qdisc = q->queues[band];
+               if (qdisc->ops->drop) {
+                       len = qdisc->ops->drop(qdisc);
+                       if (len != 0) {
+                               sch->q.qlen--;
+                               return len;
+                       }
+               }
+       }
+       return 0;
+}
+
+
+static void
+multiq_reset(struct Qdisc *sch)
+{
+       u16 band;
+       struct multiq_sched_data *q = qdisc_priv(sch);
+
+       for (band = 0; band < q->bands; band++)
+               qdisc_reset(q->queues[band]);
+       sch->q.qlen = 0;
+       q->curband = 0;
+}
+
+static void
+multiq_destroy(struct Qdisc *sch)
+{
+       int band;
+       struct multiq_sched_data *q = qdisc_priv(sch);
+
+       tcf_destroy_chain(&q->filter_list);
+       for (band = 0; band < q->bands; band++)
+               qdisc_destroy(q->queues[band]);
+
+       kfree(q->queues);
+}
+
+static int multiq_tune(struct Qdisc *sch, struct nlattr *opt)
+{
+       struct multiq_sched_data *q = qdisc_priv(sch);
+       struct tc_multiq_qopt *qopt;
+       int i;
+
+       if (!netif_is_multiqueue(qdisc_dev(sch)))
+               return -EINVAL;
+       if (nla_len(opt) < sizeof(*qopt))
+               return -EINVAL;
+
+       qopt = nla_data(opt);
+
+       qopt->bands = qdisc_dev(sch)->real_num_tx_queues;
+
+       sch_tree_lock(sch);
+       q->bands = qopt->bands;
+       for (i = q->bands; i < q->max_bands; i++) {
+               struct Qdisc *child = xchg(&q->queues[i], &noop_qdisc);
+               if (child != &noop_qdisc) {
+                       qdisc_tree_decrease_qlen(child, child->q.qlen);
+                       qdisc_destroy(child);
+               }
+       }
+
+       sch_tree_unlock(sch);
+
+       for (i = 0; i < q->bands; i++) {
+               if (q->queues[i] == &noop_qdisc) {
+                       struct Qdisc *child;
+                       child = qdisc_create_dflt(qdisc_dev(sch),
+                                                 sch->dev_queue,
+                                                 &pfifo_qdisc_ops,
+                                                 TC_H_MAKE(sch->handle,
+                                                           i + 1));
+                       if (child) {
+                               sch_tree_lock(sch);
+                               child = xchg(&q->queues[i], child);
+
+                               if (child != &noop_qdisc) {
+                                       qdisc_tree_decrease_qlen(child,
+                                                                child->q.qlen);
+                                       qdisc_destroy(child);
+                               }
+                               sch_tree_unlock(sch);
+                       }
+               }
+       }
+       return 0;
+}
+
+static int multiq_init(struct Qdisc *sch, struct nlattr *opt)
+{
+       struct multiq_sched_data *q = qdisc_priv(sch);
+       int i;
+
+       q->queues = NULL;
+
+       if (opt == NULL)
+               return -EINVAL;
+
+       q->max_bands = qdisc_dev(sch)->num_tx_queues;
+
+       q->queues = kcalloc(q->max_bands, sizeof(struct Qdisc *), GFP_KERNEL);
+       if (!q->queues)
+               return -ENOBUFS;
+       for (i = 0; i < q->max_bands; i++)
+               q->queues[i] = &noop_qdisc;
+
+       return multiq_tune(sch, opt);
+}
+
+static int multiq_dump(struct Qdisc *sch, struct sk_buff *skb)
+{
+       struct multiq_sched_data *q = qdisc_priv(sch);
+       unsigned char *b = skb_tail_pointer(skb);
+       struct tc_multiq_qopt opt;
+
+       opt.bands = q->bands;
+       opt.max_bands = q->max_bands;
+
+       NLA_PUT(skb, TCA_OPTIONS, sizeof(opt), &opt);
+
+       return skb->len;
+
+nla_put_failure:
+       nlmsg_trim(skb, b);
+       return -1;
+}
+
+static int multiq_graft(struct Qdisc *sch, unsigned long arg, struct Qdisc *new,
+                     struct Qdisc **old)
+{
+       struct multiq_sched_data *q = qdisc_priv(sch);
+       unsigned long band = arg - 1;
+
+       if (band >= q->bands)
+               return -EINVAL;
+
+       if (new == NULL)
+               new = &noop_qdisc;
+
+       sch_tree_lock(sch);
+       *old = q->queues[band];
+       q->queues[band] = new;
+       qdisc_tree_decrease_qlen(*old, (*old)->q.qlen);
+       qdisc_reset(*old);
+       sch_tree_unlock(sch);
+
+       return 0;
+}
+
+static struct Qdisc *
+multiq_leaf(struct Qdisc *sch, unsigned long arg)
+{
+       struct multiq_sched_data *q = qdisc_priv(sch);
+       unsigned long band = arg - 1;
+
+       if (band >= q->bands)
+               return NULL;
+
+       return q->queues[band];
+}
+
+static unsigned long multiq_get(struct Qdisc *sch, u32 classid)
+{
+       struct multiq_sched_data *q = qdisc_priv(sch);
+       unsigned long band = TC_H_MIN(classid);
+
+       if (band - 1 >= q->bands)
+               return 0;
+       return band;
+}
+
+static unsigned long multiq_bind(struct Qdisc *sch, unsigned long parent,
+                                u32 classid)
+{
+       return multiq_get(sch, classid);
+}
+
+
+static void multiq_put(struct Qdisc *q, unsigned long cl)
+{
+       return;
+}
+
+static int multiq_change(struct Qdisc *sch, u32 handle, u32 parent,
+                        struct nlattr **tca, unsigned long *arg)
+{
+       unsigned long cl = *arg;
+       struct multiq_sched_data *q = qdisc_priv(sch);
+
+       if (cl - 1 > q->bands)
+               return -ENOENT;
+       return 0;
+}
+
+static int multiq_delete(struct Qdisc *sch, unsigned long cl)
+{
+       struct multiq_sched_data *q = qdisc_priv(sch);
+       if (cl - 1 > q->bands)
+               return -ENOENT;
+       return 0;
+}
+
+
+static int multiq_dump_class(struct Qdisc *sch, unsigned long cl,
+                            struct sk_buff *skb, struct tcmsg *tcm)
+{
+       struct multiq_sched_data *q = qdisc_priv(sch);
+
+       if (cl - 1 > q->bands)
+               return -ENOENT;
+       tcm->tcm_handle |= TC_H_MIN(cl);
+       if (q->queues[cl-1])
+               tcm->tcm_info = q->queues[cl-1]->handle;
+       return 0;
+}
+
+static int multiq_dump_class_stats(struct Qdisc *sch, unsigned long cl,
+                                struct gnet_dump *d)
+{
+       struct multiq_sched_data *q = qdisc_priv(sch);
+       struct Qdisc *cl_q;
+
+       cl_q = q->queues[cl - 1];
+       if (gnet_stats_copy_basic(d, &cl_q->bstats) < 0 ||
+           gnet_stats_copy_queue(d, &cl_q->qstats) < 0)
+               return -1;
+
+       return 0;
+}
+
+static void multiq_walk(struct Qdisc *sch, struct qdisc_walker *arg)
+{
+       struct multiq_sched_data *q = qdisc_priv(sch);
+       int band;
+
+       if (arg->stop)
+               return;
+
+       for (band = 0; band < q->bands; band++) {
+               if (arg->count < arg->skip) {
+                       arg->count++;
+                       continue;
+               }
+               if (arg->fn(sch, band+1, arg) < 0) {
+                       arg->stop = 1;
+                       break;
+               }
+               arg->count++;
+       }
+}
+
+static struct tcf_proto **multiq_find_tcf(struct Qdisc *sch, unsigned long cl)
+{
+       struct multiq_sched_data *q = qdisc_priv(sch);
+
+       if (cl)
+               return NULL;
+       return &q->filter_list;
+}
+
+static const struct Qdisc_class_ops multiq_class_ops = {
+       .graft          =       multiq_graft,
+       .leaf           =       multiq_leaf,
+       .get            =       multiq_get,
+       .put            =       multiq_put,
+       .change         =       multiq_change,
+       .delete         =       multiq_delete,
+       .walk           =       multiq_walk,
+       .tcf_chain      =       multiq_find_tcf,
+       .bind_tcf       =       multiq_bind,
+       .unbind_tcf     =       multiq_put,
+       .dump           =       multiq_dump_class,
+       .dump_stats     =       multiq_dump_class_stats,
+};
+
+static struct Qdisc_ops multiq_qdisc_ops __read_mostly = {
+       .next           =       NULL,
+       .cl_ops         =       &multiq_class_ops,
+       .id             =       "multiq",
+       .priv_size      =       sizeof(struct multiq_sched_data),
+       .enqueue        =       multiq_enqueue,
+       .dequeue        =       multiq_dequeue,
+       .requeue        =       multiq_requeue,
+       .drop           =       multiq_drop,
+       .init           =       multiq_init,
+       .reset          =       multiq_reset,
+       .destroy        =       multiq_destroy,
+       .change         =       multiq_tune,
+       .dump           =       multiq_dump,
+       .owner          =       THIS_MODULE,
+};
+
+static int __init multiq_module_init(void)
+{
+       return register_qdisc(&multiq_qdisc_ops);
+}
+
+static void __exit multiq_module_exit(void)
+{
+       unregister_qdisc(&multiq_qdisc_ops);
+}
+
+module_init(multiq_module_init)
+module_exit(multiq_module_exit)
+
+MODULE_LICENSE("GPL");