1 #include <linux/module.h>
2 #include <linux/errno.h>
3 #include <linux/socket.h>
4 #include <linux/skbuff.h>
7 #include <linux/types.h>
8 #include <linux/kernel.h>
9 #include <net/genetlink.h>
12 #include <net/protocol.h>
14 #include <net/udp_tunnel.h>
16 #include <uapi/linux/fou.h>
17 #include <uapi/linux/genetlink.h>
25 struct udp_offload udp_offloads;
26 struct list_head list;
30 #define FOU_F_REMCSUM_NOPARTIAL BIT(0)
36 struct udp_port_cfg udp_config;
39 static unsigned int fou_net_id;
42 struct list_head fou_list;
43 struct mutex fou_lock;
46 static inline struct fou *fou_from_sock(struct sock *sk)
48 return sk->sk_user_data;
51 static int fou_recv_pull(struct sk_buff *skb, size_t len)
53 struct iphdr *iph = ip_hdr(skb);
55 /* Remove 'len' bytes from the packet (UDP header and
56 * FOU header if present).
58 iph->tot_len = htons(ntohs(iph->tot_len) - len);
60 skb_postpull_rcsum(skb, udp_hdr(skb), len);
61 skb_reset_transport_header(skb);
62 return iptunnel_pull_offloads(skb);
65 static int fou_udp_recv(struct sock *sk, struct sk_buff *skb)
67 struct fou *fou = fou_from_sock(sk);
72 if (fou_recv_pull(skb, sizeof(struct udphdr)))
75 return -fou->protocol;
82 static struct guehdr *gue_remcsum(struct sk_buff *skb, struct guehdr *guehdr,
83 void *data, size_t hdrlen, u8 ipproto,
87 size_t start = ntohs(pd[0]);
88 size_t offset = ntohs(pd[1]);
89 size_t plen = sizeof(struct udphdr) + hdrlen +
90 max_t(size_t, offset + sizeof(u16), start);
92 if (skb->remcsum_offload)
95 if (!pskb_may_pull(skb, plen))
97 guehdr = (struct guehdr *)&udp_hdr(skb)[1];
99 skb_remcsum_process(skb, (void *)guehdr + hdrlen,
100 start, offset, nopartial);
105 static int gue_control_message(struct sk_buff *skb, struct guehdr *guehdr)
112 static int gue_udp_recv(struct sock *sk, struct sk_buff *skb)
114 struct fou *fou = fou_from_sock(sk);
115 size_t len, optlen, hdrlen;
116 struct guehdr *guehdr;
123 len = sizeof(struct udphdr) + sizeof(struct guehdr);
124 if (!pskb_may_pull(skb, len))
127 guehdr = (struct guehdr *)&udp_hdr(skb)[1];
129 optlen = guehdr->hlen << 2;
132 if (!pskb_may_pull(skb, len))
135 /* guehdr may change after pull */
136 guehdr = (struct guehdr *)&udp_hdr(skb)[1];
138 hdrlen = sizeof(struct guehdr) + optlen;
140 if (guehdr->version != 0 || validate_gue_flags(guehdr, optlen))
143 hdrlen = sizeof(struct guehdr) + optlen;
145 ip_hdr(skb)->tot_len = htons(ntohs(ip_hdr(skb)->tot_len) - len);
147 /* Pull csum through the guehdr now . This can be used if
148 * there is a remote checksum offload.
150 skb_postpull_rcsum(skb, udp_hdr(skb), len);
154 if (guehdr->flags & GUE_FLAG_PRIV) {
155 __be32 flags = *(__be32 *)(data + doffset);
157 doffset += GUE_LEN_PRIV;
159 if (flags & GUE_PFLAG_REMCSUM) {
160 guehdr = gue_remcsum(skb, guehdr, data + doffset,
161 hdrlen, guehdr->proto_ctype,
163 FOU_F_REMCSUM_NOPARTIAL));
169 doffset += GUE_PLEN_REMCSUM;
173 if (unlikely(guehdr->control))
174 return gue_control_message(skb, guehdr);
176 __skb_pull(skb, sizeof(struct udphdr) + hdrlen);
177 skb_reset_transport_header(skb);
179 if (iptunnel_pull_offloads(skb))
182 return -guehdr->proto_ctype;
189 static struct sk_buff **fou_gro_receive(struct sk_buff **head,
191 struct udp_offload *uoff)
193 const struct net_offload *ops;
194 struct sk_buff **pp = NULL;
195 u8 proto = NAPI_GRO_CB(skb)->proto;
196 const struct net_offload **offloads;
199 offloads = NAPI_GRO_CB(skb)->is_ipv6 ? inet6_offloads : inet_offloads;
200 ops = rcu_dereference(offloads[proto]);
201 if (!ops || !ops->callbacks.gro_receive)
204 pp = call_gro_receive(ops->callbacks.gro_receive, head, skb);
212 static int fou_gro_complete(struct sk_buff *skb, int nhoff,
213 struct udp_offload *uoff)
215 const struct net_offload *ops;
216 u8 proto = NAPI_GRO_CB(skb)->proto;
218 const struct net_offload **offloads;
220 udp_tunnel_gro_complete(skb, nhoff);
223 offloads = NAPI_GRO_CB(skb)->is_ipv6 ? inet6_offloads : inet_offloads;
224 ops = rcu_dereference(offloads[proto]);
225 if (WARN_ON(!ops || !ops->callbacks.gro_complete))
228 err = ops->callbacks.gro_complete(skb, nhoff);
236 static struct guehdr *gue_gro_remcsum(struct sk_buff *skb, unsigned int off,
237 struct guehdr *guehdr, void *data,
238 size_t hdrlen, struct gro_remcsum *grc,
242 size_t start = ntohs(pd[0]);
243 size_t offset = ntohs(pd[1]);
245 if (skb->remcsum_offload)
248 if (!NAPI_GRO_CB(skb)->csum_valid)
251 guehdr = skb_gro_remcsum_process(skb, (void *)guehdr, off, hdrlen,
252 start, offset, grc, nopartial);
254 skb->remcsum_offload = 1;
259 static struct sk_buff **gue_gro_receive(struct sk_buff **head,
261 struct udp_offload *uoff)
263 const struct net_offload **offloads;
264 const struct net_offload *ops;
265 struct sk_buff **pp = NULL;
267 struct guehdr *guehdr;
268 size_t len, optlen, hdrlen, off;
272 struct fou *fou = container_of(uoff, struct fou, udp_offloads);
273 struct gro_remcsum grc;
275 skb_gro_remcsum_init(&grc);
277 off = skb_gro_offset(skb);
278 len = off + sizeof(*guehdr);
280 guehdr = skb_gro_header_fast(skb, off);
281 if (skb_gro_header_hard(skb, len)) {
282 guehdr = skb_gro_header_slow(skb, len, off);
283 if (unlikely(!guehdr))
287 optlen = guehdr->hlen << 2;
290 if (skb_gro_header_hard(skb, len)) {
291 guehdr = skb_gro_header_slow(skb, len, off);
292 if (unlikely(!guehdr))
296 if (unlikely(guehdr->control) || guehdr->version != 0 ||
297 validate_gue_flags(guehdr, optlen))
300 hdrlen = sizeof(*guehdr) + optlen;
302 /* Adjust NAPI_GRO_CB(skb)->csum to account for guehdr,
303 * this is needed if there is a remote checkcsum offload.
305 skb_gro_postpull_rcsum(skb, guehdr, hdrlen);
309 if (guehdr->flags & GUE_FLAG_PRIV) {
310 __be32 flags = *(__be32 *)(data + doffset);
312 doffset += GUE_LEN_PRIV;
314 if (flags & GUE_PFLAG_REMCSUM) {
315 guehdr = gue_gro_remcsum(skb, off, guehdr,
316 data + doffset, hdrlen, &grc,
318 FOU_F_REMCSUM_NOPARTIAL));
325 doffset += GUE_PLEN_REMCSUM;
329 skb_gro_pull(skb, hdrlen);
333 for (p = *head; p; p = p->next) {
334 const struct guehdr *guehdr2;
336 if (!NAPI_GRO_CB(p)->same_flow)
339 guehdr2 = (struct guehdr *)(p->data + off);
341 /* Compare base GUE header to be equal (covers
342 * hlen, version, proto_ctype, and flags.
344 if (guehdr->word != guehdr2->word) {
345 NAPI_GRO_CB(p)->same_flow = 0;
349 /* Compare optional fields are the same. */
350 if (guehdr->hlen && memcmp(&guehdr[1], &guehdr2[1],
351 guehdr->hlen << 2)) {
352 NAPI_GRO_CB(p)->same_flow = 0;
358 offloads = NAPI_GRO_CB(skb)->is_ipv6 ? inet6_offloads : inet_offloads;
359 ops = rcu_dereference(offloads[guehdr->proto_ctype]);
360 if (WARN_ON_ONCE(!ops || !ops->callbacks.gro_receive))
363 pp = call_gro_receive(ops->callbacks.gro_receive, head, skb);
368 NAPI_GRO_CB(skb)->flush |= flush;
369 skb_gro_remcsum_cleanup(skb, &grc);
374 static int gue_gro_complete(struct sk_buff *skb, int nhoff,
375 struct udp_offload *uoff)
377 const struct net_offload **offloads;
378 struct guehdr *guehdr = (struct guehdr *)(skb->data + nhoff);
379 const struct net_offload *ops;
380 unsigned int guehlen;
384 proto = guehdr->proto_ctype;
386 guehlen = sizeof(*guehdr) + (guehdr->hlen << 2);
389 offloads = NAPI_GRO_CB(skb)->is_ipv6 ? inet6_offloads : inet_offloads;
390 ops = rcu_dereference(offloads[proto]);
391 if (WARN_ON(!ops || !ops->callbacks.gro_complete))
394 err = ops->callbacks.gro_complete(skb, nhoff + guehlen);
401 static int fou_add_to_port_list(struct net *net, struct fou *fou)
403 struct fou_net *fn = net_generic(net, fou_net_id);
406 mutex_lock(&fn->fou_lock);
407 list_for_each_entry(fout, &fn->fou_list, list) {
408 if (fou->port == fout->port) {
409 mutex_unlock(&fn->fou_lock);
414 list_add(&fou->list, &fn->fou_list);
415 mutex_unlock(&fn->fou_lock);
420 static void fou_release(struct fou *fou)
422 struct socket *sock = fou->sock;
423 struct sock *sk = sock->sk;
425 if (sk->sk_family == AF_INET)
426 udp_del_offload(&fou->udp_offloads);
427 list_del(&fou->list);
428 udp_tunnel_sock_release(sock);
433 static int fou_encap_init(struct sock *sk, struct fou *fou, struct fou_cfg *cfg)
435 udp_sk(sk)->encap_rcv = fou_udp_recv;
436 fou->protocol = cfg->protocol;
437 fou->udp_offloads.callbacks.gro_receive = fou_gro_receive;
438 fou->udp_offloads.callbacks.gro_complete = fou_gro_complete;
439 fou->udp_offloads.port = cfg->udp_config.local_udp_port;
440 fou->udp_offloads.ipproto = cfg->protocol;
445 static int gue_encap_init(struct sock *sk, struct fou *fou, struct fou_cfg *cfg)
447 udp_sk(sk)->encap_rcv = gue_udp_recv;
448 fou->udp_offloads.callbacks.gro_receive = gue_gro_receive;
449 fou->udp_offloads.callbacks.gro_complete = gue_gro_complete;
450 fou->udp_offloads.port = cfg->udp_config.local_udp_port;
455 static int fou_create(struct net *net, struct fou_cfg *cfg,
456 struct socket **sockp)
458 struct socket *sock = NULL;
459 struct fou *fou = NULL;
463 /* Open UDP socket */
464 err = udp_sock_create(net, &cfg->udp_config, &sock);
468 /* Allocate FOU port structure */
469 fou = kzalloc(sizeof(*fou), GFP_KERNEL);
477 fou->flags = cfg->flags;
478 fou->port = cfg->udp_config.local_udp_port;
480 /* Initial for fou type */
482 case FOU_ENCAP_DIRECT:
483 err = fou_encap_init(sk, fou, cfg);
488 err = gue_encap_init(sk, fou, cfg);
497 fou->type = cfg->type;
499 udp_sk(sk)->encap_type = 1;
502 sk->sk_user_data = fou;
505 inet_inc_convert_csum(sk);
507 sk->sk_allocation = GFP_ATOMIC;
509 if (cfg->udp_config.family == AF_INET) {
510 err = udp_add_offload(&fou->udp_offloads);
515 err = fou_add_to_port_list(net, fou);
527 udp_tunnel_sock_release(sock);
532 static int fou_destroy(struct net *net, struct fou_cfg *cfg)
534 struct fou_net *fn = net_generic(net, fou_net_id);
535 __be16 port = cfg->udp_config.local_udp_port;
539 mutex_lock(&fn->fou_lock);
540 list_for_each_entry(fou, &fn->fou_list, list) {
541 if (fou->port == port) {
547 mutex_unlock(&fn->fou_lock);
552 static struct genl_family fou_nl_family = {
553 .id = GENL_ID_GENERATE,
555 .name = FOU_GENL_NAME,
556 .version = FOU_GENL_VERSION,
557 .maxattr = FOU_ATTR_MAX,
561 static struct nla_policy fou_nl_policy[FOU_ATTR_MAX + 1] = {
562 [FOU_ATTR_PORT] = { .type = NLA_U16, },
563 [FOU_ATTR_AF] = { .type = NLA_U8, },
564 [FOU_ATTR_IPPROTO] = { .type = NLA_U8, },
565 [FOU_ATTR_TYPE] = { .type = NLA_U8, },
566 [FOU_ATTR_REMCSUM_NOPARTIAL] = { .type = NLA_FLAG, },
569 static int parse_nl_config(struct genl_info *info,
572 memset(cfg, 0, sizeof(*cfg));
574 cfg->udp_config.family = AF_INET;
576 if (info->attrs[FOU_ATTR_AF]) {
577 u8 family = nla_get_u8(info->attrs[FOU_ATTR_AF]);
579 if (family != AF_INET)
582 cfg->udp_config.family = family;
585 if (info->attrs[FOU_ATTR_PORT]) {
586 __be16 port = nla_get_be16(info->attrs[FOU_ATTR_PORT]);
588 cfg->udp_config.local_udp_port = port;
591 if (info->attrs[FOU_ATTR_IPPROTO])
592 cfg->protocol = nla_get_u8(info->attrs[FOU_ATTR_IPPROTO]);
594 if (info->attrs[FOU_ATTR_TYPE])
595 cfg->type = nla_get_u8(info->attrs[FOU_ATTR_TYPE]);
597 if (info->attrs[FOU_ATTR_REMCSUM_NOPARTIAL])
598 cfg->flags |= FOU_F_REMCSUM_NOPARTIAL;
603 static int fou_nl_cmd_add_port(struct sk_buff *skb, struct genl_info *info)
605 struct net *net = genl_info_net(info);
609 err = parse_nl_config(info, &cfg);
613 return fou_create(net, &cfg, NULL);
616 static int fou_nl_cmd_rm_port(struct sk_buff *skb, struct genl_info *info)
618 struct net *net = genl_info_net(info);
622 err = parse_nl_config(info, &cfg);
626 return fou_destroy(net, &cfg);
629 static int fou_fill_info(struct fou *fou, struct sk_buff *msg)
631 if (nla_put_u8(msg, FOU_ATTR_AF, fou->sock->sk->sk_family) ||
632 nla_put_be16(msg, FOU_ATTR_PORT, fou->port) ||
633 nla_put_u8(msg, FOU_ATTR_IPPROTO, fou->protocol) ||
634 nla_put_u8(msg, FOU_ATTR_TYPE, fou->type))
637 if (fou->flags & FOU_F_REMCSUM_NOPARTIAL)
638 if (nla_put_flag(msg, FOU_ATTR_REMCSUM_NOPARTIAL))
643 static int fou_dump_info(struct fou *fou, u32 portid, u32 seq,
644 u32 flags, struct sk_buff *skb, u8 cmd)
648 hdr = genlmsg_put(skb, portid, seq, &fou_nl_family, flags, cmd);
652 if (fou_fill_info(fou, skb) < 0)
653 goto nla_put_failure;
655 genlmsg_end(skb, hdr);
659 genlmsg_cancel(skb, hdr);
663 static int fou_nl_cmd_get_port(struct sk_buff *skb, struct genl_info *info)
665 struct net *net = genl_info_net(info);
666 struct fou_net *fn = net_generic(net, fou_net_id);
673 ret = parse_nl_config(info, &cfg);
676 port = cfg.udp_config.local_udp_port;
680 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
685 mutex_lock(&fn->fou_lock);
686 list_for_each_entry(fout, &fn->fou_list, list) {
687 if (port == fout->port) {
688 ret = fou_dump_info(fout, info->snd_portid,
689 info->snd_seq, 0, msg,
694 mutex_unlock(&fn->fou_lock);
698 return genlmsg_reply(msg, info);
705 static int fou_nl_dump(struct sk_buff *skb, struct netlink_callback *cb)
707 struct net *net = sock_net(skb->sk);
708 struct fou_net *fn = net_generic(net, fou_net_id);
712 mutex_lock(&fn->fou_lock);
713 list_for_each_entry(fout, &fn->fou_list, list) {
714 if (idx++ < cb->args[0])
716 ret = fou_dump_info(fout, NETLINK_CB(cb->skb).portid,
717 cb->nlh->nlmsg_seq, NLM_F_MULTI,
722 mutex_unlock(&fn->fou_lock);
728 static const struct genl_ops fou_nl_ops[] = {
731 .doit = fou_nl_cmd_add_port,
732 .policy = fou_nl_policy,
733 .flags = GENL_ADMIN_PERM,
737 .doit = fou_nl_cmd_rm_port,
738 .policy = fou_nl_policy,
739 .flags = GENL_ADMIN_PERM,
743 .doit = fou_nl_cmd_get_port,
744 .dumpit = fou_nl_dump,
745 .policy = fou_nl_policy,
749 size_t fou_encap_hlen(struct ip_tunnel_encap *e)
751 return sizeof(struct udphdr);
753 EXPORT_SYMBOL(fou_encap_hlen);
755 size_t gue_encap_hlen(struct ip_tunnel_encap *e)
758 bool need_priv = false;
760 len = sizeof(struct udphdr) + sizeof(struct guehdr);
762 if (e->flags & TUNNEL_ENCAP_FLAG_REMCSUM) {
763 len += GUE_PLEN_REMCSUM;
767 len += need_priv ? GUE_LEN_PRIV : 0;
771 EXPORT_SYMBOL(gue_encap_hlen);
773 static void fou_build_udp(struct sk_buff *skb, struct ip_tunnel_encap *e,
774 struct flowi4 *fl4, u8 *protocol, __be16 sport)
778 skb_push(skb, sizeof(struct udphdr));
779 skb_reset_transport_header(skb);
785 uh->len = htons(skb->len);
787 udp_set_csum(!(e->flags & TUNNEL_ENCAP_FLAG_CSUM), skb,
788 fl4->saddr, fl4->daddr, skb->len);
790 *protocol = IPPROTO_UDP;
793 int fou_build_header(struct sk_buff *skb, struct ip_tunnel_encap *e,
794 u8 *protocol, struct flowi4 *fl4)
796 bool csum = !!(e->flags & TUNNEL_ENCAP_FLAG_CSUM);
797 int type = csum ? SKB_GSO_UDP_TUNNEL_CSUM : SKB_GSO_UDP_TUNNEL;
800 skb = iptunnel_handle_offloads(skb, csum, type);
805 sport = e->sport ? : udp_flow_src_port(dev_net(skb->dev),
807 fou_build_udp(skb, e, fl4, protocol, sport);
811 EXPORT_SYMBOL(fou_build_header);
813 int gue_build_header(struct sk_buff *skb, struct ip_tunnel_encap *e,
814 u8 *protocol, struct flowi4 *fl4)
816 bool csum = !!(e->flags & TUNNEL_ENCAP_FLAG_CSUM);
817 int type = csum ? SKB_GSO_UDP_TUNNEL_CSUM : SKB_GSO_UDP_TUNNEL;
818 struct guehdr *guehdr;
819 size_t hdrlen, optlen = 0;
822 bool need_priv = false;
824 if ((e->flags & TUNNEL_ENCAP_FLAG_REMCSUM) &&
825 skb->ip_summed == CHECKSUM_PARTIAL) {
827 optlen += GUE_PLEN_REMCSUM;
828 type |= SKB_GSO_TUNNEL_REMCSUM;
832 optlen += need_priv ? GUE_LEN_PRIV : 0;
834 skb = iptunnel_handle_offloads(skb, csum, type);
839 /* Get source port (based on flow hash) before skb_push */
840 sport = e->sport ? : udp_flow_src_port(dev_net(skb->dev),
843 hdrlen = sizeof(struct guehdr) + optlen;
845 skb_push(skb, hdrlen);
847 guehdr = (struct guehdr *)skb->data;
851 guehdr->hlen = optlen >> 2;
853 guehdr->proto_ctype = *protocol;
858 __be32 *flags = data;
860 guehdr->flags |= GUE_FLAG_PRIV;
862 data += GUE_LEN_PRIV;
864 if (type & SKB_GSO_TUNNEL_REMCSUM) {
865 u16 csum_start = skb_checksum_start_offset(skb);
868 if (csum_start < hdrlen)
871 csum_start -= hdrlen;
872 pd[0] = htons(csum_start);
873 pd[1] = htons(csum_start + skb->csum_offset);
875 if (!skb_is_gso(skb)) {
876 skb->ip_summed = CHECKSUM_NONE;
877 skb->encapsulation = 0;
880 *flags |= GUE_PFLAG_REMCSUM;
881 data += GUE_PLEN_REMCSUM;
886 fou_build_udp(skb, e, fl4, protocol, sport);
890 EXPORT_SYMBOL(gue_build_header);
892 #ifdef CONFIG_NET_FOU_IP_TUNNELS
894 static const struct ip_tunnel_encap_ops fou_iptun_ops = {
895 .encap_hlen = fou_encap_hlen,
896 .build_header = fou_build_header,
899 static const struct ip_tunnel_encap_ops gue_iptun_ops = {
900 .encap_hlen = gue_encap_hlen,
901 .build_header = gue_build_header,
904 static int ip_tunnel_encap_add_fou_ops(void)
908 ret = ip_tunnel_encap_add_ops(&fou_iptun_ops, TUNNEL_ENCAP_FOU);
910 pr_err("can't add fou ops\n");
914 ret = ip_tunnel_encap_add_ops(&gue_iptun_ops, TUNNEL_ENCAP_GUE);
916 pr_err("can't add gue ops\n");
917 ip_tunnel_encap_del_ops(&fou_iptun_ops, TUNNEL_ENCAP_FOU);
924 static void ip_tunnel_encap_del_fou_ops(void)
926 ip_tunnel_encap_del_ops(&fou_iptun_ops, TUNNEL_ENCAP_FOU);
927 ip_tunnel_encap_del_ops(&gue_iptun_ops, TUNNEL_ENCAP_GUE);
932 static int ip_tunnel_encap_add_fou_ops(void)
937 static void ip_tunnel_encap_del_fou_ops(void)
943 static __net_init int fou_init_net(struct net *net)
945 struct fou_net *fn = net_generic(net, fou_net_id);
947 INIT_LIST_HEAD(&fn->fou_list);
948 mutex_init(&fn->fou_lock);
952 static __net_exit void fou_exit_net(struct net *net)
954 struct fou_net *fn = net_generic(net, fou_net_id);
955 struct fou *fou, *next;
957 /* Close all the FOU sockets */
958 mutex_lock(&fn->fou_lock);
959 list_for_each_entry_safe(fou, next, &fn->fou_list, list)
961 mutex_unlock(&fn->fou_lock);
964 static struct pernet_operations fou_net_ops = {
965 .init = fou_init_net,
966 .exit = fou_exit_net,
968 .size = sizeof(struct fou_net),
971 static int __init fou_init(void)
975 ret = register_pernet_device(&fou_net_ops);
979 ret = genl_register_family_with_ops(&fou_nl_family,
984 ret = ip_tunnel_encap_add_fou_ops();
988 genl_unregister_family(&fou_nl_family);
990 unregister_pernet_device(&fou_net_ops);
995 static void __exit fou_fini(void)
997 ip_tunnel_encap_del_fou_ops();
998 genl_unregister_family(&fou_nl_family);
999 unregister_pernet_device(&fou_net_ops);
1002 module_init(fou_init);
1003 module_exit(fou_fini);
1004 MODULE_AUTHOR("Tom Herbert <therbert@google.com>");
1005 MODULE_LICENSE("GPL");