net/core/flow_dissector.c

   1 #include <linux/kernel.h>
   2 #include <linux/skbuff.h>
   3 #include <linux/export.h>
   4 #include <linux/ip.h>
   5 #include <linux/ipv6.h>
   6 #include <linux/if_vlan.h>
   7 #include <net/ip.h>
   8 #include <net/ipv6.h>
   9 #include <linux/igmp.h>
  10 #include <linux/icmp.h>
  11 #include <linux/sctp.h>
  12 #include <linux/dccp.h>
  13 #include <linux/if_tunnel.h>
  14 #include <linux/if_pppox.h>
  15 #include <linux/ppp_defs.h>
  16 #include <linux/stddef.h>
  17 #include <net/flow_dissector.h>
  18 #include <scsi/fc/fc_fcoe.h>
  19
  20 static bool skb_flow_dissector_uses_key(struct flow_dissector *flow_dissector,
  21                                         enum flow_dissector_key_id key_id)
  22 {
  23         return flow_dissector->used_keys & (1 << key_id);
  24 }
  25
  26 static void skb_flow_dissector_set_key(struct flow_dissector *flow_dissector,
  27                                        enum flow_dissector_key_id key_id)
  28 {
  29         flow_dissector->used_keys |= (1 << key_id);
  30 }
  31
  32 static void *skb_flow_dissector_target(struct flow_dissector *flow_dissector,
  33                                        enum flow_dissector_key_id key_id,
  34                                        void *target_container)
  35 {
  36         return ((char *) target_container) + flow_dissector->offset[key_id];
  37 }
  38
  39 void skb_flow_dissector_init(struct flow_dissector *flow_dissector,
  40                              const struct flow_dissector_key *key,
  41                              unsigned int key_count)
  42 {
  43         unsigned int i;
  44
  45         memset(flow_dissector, 0, sizeof(*flow_dissector));
  46
  47         for (i = 0; i < key_count; i++, key++) {
  48                 /* User should make sure that every key target offset is withing
  49                  * boundaries of unsigned short.
  50                  */
  51                 BUG_ON(key->offset > USHRT_MAX);
  52                 BUG_ON(skb_flow_dissector_uses_key(flow_dissector,
  53                                                    key->key_id));
  54
  55                 skb_flow_dissector_set_key(flow_dissector, key->key_id);
  56                 flow_dissector->offset[key->key_id] = key->offset;
  57         }
  58
  59         /* Ensure that the dissector always includes basic key. That way
  60          * we are able to avoid handling lack of it in fast path.
  61          */
  62         BUG_ON(!skb_flow_dissector_uses_key(flow_dissector,
  63                                             FLOW_DISSECTOR_KEY_BASIC));
  64 }
  65 EXPORT_SYMBOL(skb_flow_dissector_init);
  66
  67 /**
  68  * __skb_flow_get_ports - extract the upper layer ports and return them
  69  * @skb: sk_buff to extract the ports from
  70  * @thoff: transport header offset
  71  * @ip_proto: protocol for which to get port offset
  72  * @data: raw buffer pointer to the packet, if NULL use skb->data
  73  * @hlen: packet header length, if @data is NULL use skb_headlen(skb)
  74  *
  75  * The function will try to retrieve the ports at offset thoff + poff where poff
  76  * is the protocol port offset returned from proto_ports_offset
  77  */
  78 __be32 __skb_flow_get_ports(const struct sk_buff *skb, int thoff, u8 ip_proto,
  79                             void *data, int hlen)
  80 {
  81         int poff = proto_ports_offset(ip_proto);
  82
  83         if (!data) {
  84                 data = skb->data;
  85                 hlen = skb_headlen(skb);
  86         }
  87
  88         if (poff >= 0) {
  89                 __be32 *ports, _ports;
  90
  91                 ports = __skb_header_pointer(skb, thoff + poff,
  92                                              sizeof(_ports), data, hlen, &_ports);
  93                 if (ports)
  94                         return *ports;
  95         }
  96
  97         return 0;
  98 }
  99 EXPORT_SYMBOL(__skb_flow_get_ports);
 100
 101 /**
 102  * __skb_flow_dissect - extract the flow_keys struct and return it
 103  * @skb: sk_buff to extract the flow from, can be NULL if the rest are specified
 104  * @flow_dissector: list of keys to dissect
 105  * @target_container: target structure to put dissected values into
 106  * @data: raw buffer pointer to the packet, if NULL use skb->data
 107  * @proto: protocol for which to get the flow, if @data is NULL use skb->protocol
 108  * @nhoff: network header offset, if @data is NULL use skb_network_offset(skb)
 109  * @hlen: packet header length, if @data is NULL use skb_headlen(skb)
 110  *
 111  * The function will try to retrieve individual keys into target specified
 112  * by flow_dissector from either the skbuff or a raw buffer specified by the
 113  * rest parameters.
 114  *
 115  * Caller must take care of zeroing target container memory.
 116  */
 117 bool __skb_flow_dissect(const struct sk_buff *skb,
 118                         struct flow_dissector *flow_dissector,
 119                         void *target_container,
 120                         void *data, __be16 proto, int nhoff, int hlen)
 121 {
 122         struct flow_dissector_key_basic *key_basic;
 123         struct flow_dissector_key_addrs *key_addrs;
 124         struct flow_dissector_key_ports *key_ports;
 125         u8 ip_proto;
 126
 127         if (!data) {
 128                 data = skb->data;
 129                 proto = skb->protocol;
 130                 nhoff = skb_network_offset(skb);
 131                 hlen = skb_headlen(skb);
 132         }
 133
 134         /* It is ensured by skb_flow_dissector_init() that basic key will
 135          * be always present.
 136          */
 137         key_basic = skb_flow_dissector_target(flow_dissector,
 138                                               FLOW_DISSECTOR_KEY_BASIC,
 139                                               target_container);
 140
 141 again:
 142         switch (proto) {
 143         case htons(ETH_P_IP): {
 144                 const struct iphdr *iph;
 145                 struct iphdr _iph;
 146 ip:
 147                 iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph);
 148                 if (!iph || iph->ihl < 5)
 149                         return false;
 150                 nhoff += iph->ihl * 4;
 151
 152                 ip_proto = iph->protocol;
 153                 if (ip_is_fragment(iph))
 154                         ip_proto = 0;
 155
 156                 if (!skb_flow_dissector_uses_key(flow_dissector,
 157                                                  FLOW_DISSECTOR_KEY_IPV4_ADDRS))
 158                         break;
 159                 key_addrs = skb_flow_dissector_target(flow_dissector,
 160                                                       FLOW_DISSECTOR_KEY_IPV4_ADDRS,
 161                                                       target_container);
 162                 memcpy(key_addrs, &iph->saddr, sizeof(*key_addrs));
 163                 break;
 164         }
 165         case htons(ETH_P_IPV6): {
 166                 const struct ipv6hdr *iph;
 167                 struct ipv6hdr _iph;
 168                 __be32 flow_label;
 169
 170 ipv6:
 171                 iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph);
 172                 if (!iph)
 173                         return false;
 174
 175                 ip_proto = iph->nexthdr;
 176                 nhoff += sizeof(struct ipv6hdr);
 177
 178                 if (skb_flow_dissector_uses_key(flow_dissector,
 179                                                 FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS)) {
 180                         key_addrs = skb_flow_dissector_target(flow_dissector,
 181                                                               FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS,
 182                                                               target_container);
 183
 184                         key_addrs->src = (__force __be32)ipv6_addr_hash(&iph->saddr);
 185                         key_addrs->dst = (__force __be32)ipv6_addr_hash(&iph->daddr);
 186                         goto flow_label;
 187                 }
 188                 if (skb_flow_dissector_uses_key(flow_dissector,
 189                                                 FLOW_DISSECTOR_KEY_IPV6_ADDRS)) {
 190                         struct flow_dissector_key_ipv6_addrs *key_ipv6_addrs;
 191
 192                         key_ipv6_addrs = skb_flow_dissector_target(flow_dissector,
 193                                                                    FLOW_DISSECTOR_KEY_IPV6_ADDRS,
 194                                                                    target_container);
 195
 196                         memcpy(key_ipv6_addrs, &iph->saddr, sizeof(*key_ipv6_addrs));
 197                         goto flow_label;
 198                 }
 199                 break;
 200 flow_label:
 201                 flow_label = ip6_flowlabel(iph);
 202                 if (flow_label) {
 203                         /* Awesome, IPv6 packet has a flow label so we can
 204                          * use that to represent the ports without any
 205                          * further dissection.
 206                          */
 207
 208                         key_basic->n_proto = proto;
 209                         key_basic->ip_proto = ip_proto;
 210                         key_basic->thoff = (u16)nhoff;
 211
 212                         if (!skb_flow_dissector_uses_key(flow_dissector,
 213                                                          FLOW_DISSECTOR_KEY_PORTS))
 214                                 break;
 215                         key_ports = skb_flow_dissector_target(flow_dissector,
 216                                                               FLOW_DISSECTOR_KEY_PORTS,
 217                                                               target_container);
 218                         key_ports->ports = flow_label;
 219
 220                         return true;
 221                 }
 222
 223                 break;
 224         }
 225         case htons(ETH_P_8021AD):
 226         case htons(ETH_P_8021Q): {
 227                 const struct vlan_hdr *vlan;
 228                 struct vlan_hdr _vlan;
 229
 230                 vlan = __skb_header_pointer(skb, nhoff, sizeof(_vlan), data, hlen, &_vlan);
 231                 if (!vlan)
 232                         return false;
 233
 234                 proto = vlan->h_vlan_encapsulated_proto;
 235                 nhoff += sizeof(*vlan);
 236                 goto again;
 237         }
 238         case htons(ETH_P_PPP_SES): {
 239                 struct {
 240                         struct pppoe_hdr hdr;
 241                         __be16 proto;
 242                 } *hdr, _hdr;
 243                 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
 244                 if (!hdr)
 245                         return false;
 246                 proto = hdr->proto;
 247                 nhoff += PPPOE_SES_HLEN;
 248                 switch (proto) {
 249                 case htons(PPP_IP):
 250                         goto ip;
 251                 case htons(PPP_IPV6):
 252                         goto ipv6;
 253                 default:
 254                         return false;
 255                 }
 256         }
 257         case htons(ETH_P_TIPC): {
 258                 struct {
 259                         __be32 pre[3];
 260                         __be32 srcnode;
 261                 } *hdr, _hdr;
 262                 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
 263                 if (!hdr)
 264                         return false;
 265                 key_basic->n_proto = proto;
 266                 key_basic->thoff = (u16)nhoff;
 267
 268                 if (skb_flow_dissector_uses_key(flow_dissector,
 269                                                 FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS)) {
 270                         return true;
 271                         key_addrs = skb_flow_dissector_target(flow_dissector,
 272                                                               FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS,
 273                                                               target_container);
 274                         key_addrs->src = hdr->srcnode;
 275                         key_addrs->dst = 0;
 276                 }
 277                 return true;
 278         }
 279         case htons(ETH_P_FCOE):
 280                 key_basic->thoff = (u16)(nhoff + FCOE_HEADER_LEN);
 281                 /* fall through */
 282         default:
 283                 return false;
 284         }
 285
 286         switch (ip_proto) {
 287         case IPPROTO_GRE: {
 288                 struct gre_hdr {
 289                         __be16 flags;
 290                         __be16 proto;
 291                 } *hdr, _hdr;
 292
 293                 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
 294                 if (!hdr)
 295                         return false;
 296                 /*
 297                  * Only look inside GRE if version zero and no
 298                  * routing
 299                  */
 300                 if (!(hdr->flags & (GRE_VERSION|GRE_ROUTING))) {
 301                         proto = hdr->proto;
 302                         nhoff += 4;
 303                         if (hdr->flags & GRE_CSUM)
 304                                 nhoff += 4;
 305                         if (hdr->flags & GRE_KEY)
 306                                 nhoff += 4;
 307                         if (hdr->flags & GRE_SEQ)
 308                                 nhoff += 4;
 309                         if (proto == htons(ETH_P_TEB)) {
 310                                 const struct ethhdr *eth;
 311                                 struct ethhdr _eth;
 312
 313                                 eth = __skb_header_pointer(skb, nhoff,
 314                                                            sizeof(_eth),
 315                                                            data, hlen, &_eth);
 316                                 if (!eth)
 317                                         return false;
 318                                 proto = eth->h_proto;
 319                                 nhoff += sizeof(*eth);
 320                         }
 321                         goto again;
 322                 }
 323                 break;
 324         }
 325         case IPPROTO_IPIP:
 326                 proto = htons(ETH_P_IP);
 327                 goto ip;
 328         case IPPROTO_IPV6:
 329                 proto = htons(ETH_P_IPV6);
 330                 goto ipv6;
 331         default:
 332                 break;
 333         }
 334
 335         /* It is ensured by skb_flow_dissector_init() that basic key will
 336          * be always present.
 337          */
 338         key_basic = skb_flow_dissector_target(flow_dissector,
 339                                               FLOW_DISSECTOR_KEY_BASIC,
 340                                               target_container);
 341         key_basic->n_proto = proto;
 342         key_basic->ip_proto = ip_proto;
 343         key_basic->thoff = (u16) nhoff;
 344
 345         if (skb_flow_dissector_uses_key(flow_dissector,
 346                                         FLOW_DISSECTOR_KEY_PORTS)) {
 347                 key_ports = skb_flow_dissector_target(flow_dissector,
 348                                                       FLOW_DISSECTOR_KEY_PORTS,
 349                                                       target_container);
 350                 key_ports->ports = __skb_flow_get_ports(skb, nhoff, ip_proto,
 351                                                         data, hlen);
 352         }
 353
 354         return true;
 355 }
 356 EXPORT_SYMBOL(__skb_flow_dissect);
 357
 358 static u32 hashrnd __read_mostly;
 359 static __always_inline void __flow_hash_secret_init(void)
 360 {
 361         net_get_random_once(&hashrnd, sizeof(hashrnd));
 362 }
 363
 364 static __always_inline u32 __flow_hash_3words(u32 a, u32 b, u32 c, u32 keyval)
 365 {
 366         return jhash_3words(a, b, c, keyval);
 367 }
 368
 369 static inline u32 __flow_hash_from_keys(struct flow_keys *keys, u32 keyval)
 370 {
 371         u32 hash;
 372
 373         /* get a consistent hash (same value on both flow directions) */
 374         if (((__force u32)keys->addrs.dst < (__force u32)keys->addrs.src) ||
 375             (((__force u32)keys->addrs.dst == (__force u32)keys->addrs.src) &&
 376              ((__force u16)keys->ports.port16[1] < (__force u16)keys->ports.port16[0]))) {
 377                 swap(keys->addrs.dst, keys->addrs.src);
 378                 swap(keys->ports.port16[0], keys->ports.port16[1]);
 379         }
 380
 381         hash = __flow_hash_3words((__force u32)keys->addrs.dst,
 382                                   (__force u32)keys->addrs.src,
 383                                   (__force u32)keys->ports.ports,
 384                                   keyval);
 385         if (!hash)
 386                 hash = 1;
 387
 388         return hash;
 389 }
 390
 391 u32 flow_hash_from_keys(struct flow_keys *keys)
 392 {
 393         __flow_hash_secret_init();
 394         return __flow_hash_from_keys(keys, hashrnd);
 395 }
 396 EXPORT_SYMBOL(flow_hash_from_keys);
 397
 398 static inline u32 ___skb_get_hash(const struct sk_buff *skb,
 399                                   struct flow_keys *keys, u32 keyval)
 400 {
 401         if (!skb_flow_dissect_flow_keys(skb, keys))
 402                 return 0;
 403
 404         return __flow_hash_from_keys(keys, keyval);
 405 }
 406
 407 struct _flow_keys_digest_data {
 408         __be16  n_proto;
 409         u8      ip_proto;
 410         u8      padding;
 411         __be32  ports;
 412         __be32  src;
 413         __be32  dst;
 414 };
 415
 416 void make_flow_keys_digest(struct flow_keys_digest *digest,
 417                            const struct flow_keys *flow)
 418 {
 419         struct _flow_keys_digest_data *data =
 420             (struct _flow_keys_digest_data *)digest;
 421
 422         BUILD_BUG_ON(sizeof(*data) > sizeof(*digest));
 423
 424         memset(digest, 0, sizeof(*digest));
 425
 426         data->n_proto = flow->basic.n_proto;
 427         data->ip_proto = flow->basic.ip_proto;
 428         data->ports = flow->ports.ports;
 429         data->src = flow->addrs.src;
 430         data->dst = flow->addrs.dst;
 431 }
 432 EXPORT_SYMBOL(make_flow_keys_digest);
 433
 434 /**
 435  * __skb_get_hash: calculate a flow hash
 436  * @skb: sk_buff to calculate flow hash from
 437  *
 438  * This function calculates a flow hash based on src/dst addresses
 439  * and src/dst port numbers.  Sets hash in skb to non-zero hash value
 440  * on success, zero indicates no valid hash.  Also, sets l4_hash in skb
 441  * if hash is a canonical 4-tuple hash over transport ports.
 442  */
 443 void __skb_get_hash(struct sk_buff *skb)
 444 {
 445         struct flow_keys keys;
 446         u32 hash;
 447
 448         __flow_hash_secret_init();
 449
 450         hash = ___skb_get_hash(skb, &keys, hashrnd);
 451         if (!hash)
 452                 return;
 453         if (keys.ports.ports)
 454                 skb->l4_hash = 1;
 455         skb->sw_hash = 1;
 456         skb->hash = hash;
 457 }
 458 EXPORT_SYMBOL(__skb_get_hash);
 459
 460 __u32 skb_get_hash_perturb(const struct sk_buff *skb, u32 perturb)
 461 {
 462         struct flow_keys keys;
 463
 464         return ___skb_get_hash(skb, &keys, perturb);
 465 }
 466 EXPORT_SYMBOL(skb_get_hash_perturb);
 467
 468 u32 __skb_get_poff(const struct sk_buff *skb, void *data,
 469                    const struct flow_keys *keys, int hlen)
 470 {
 471         u32 poff = keys->basic.thoff;
 472
 473         switch (keys->basic.ip_proto) {
 474         case IPPROTO_TCP: {
 475                 /* access doff as u8 to avoid unaligned access */
 476                 const u8 *doff;
 477                 u8 _doff;
 478
 479                 doff = __skb_header_pointer(skb, poff + 12, sizeof(_doff),
 480                                             data, hlen, &_doff);
 481                 if (!doff)
 482                         return poff;
 483
 484                 poff += max_t(u32, sizeof(struct tcphdr), (*doff & 0xF0) >> 2);
 485                 break;
 486         }
 487         case IPPROTO_UDP:
 488         case IPPROTO_UDPLITE:
 489                 poff += sizeof(struct udphdr);
 490                 break;
 491         /* For the rest, we do not really care about header
 492          * extensions at this point for now.
 493          */
 494         case IPPROTO_ICMP:
 495                 poff += sizeof(struct icmphdr);
 496                 break;
 497         case IPPROTO_ICMPV6:
 498                 poff += sizeof(struct icmp6hdr);
 499                 break;
 500         case IPPROTO_IGMP:
 501                 poff += sizeof(struct igmphdr);
 502                 break;
 503         case IPPROTO_DCCP:
 504                 poff += sizeof(struct dccp_hdr);
 505                 break;
 506         case IPPROTO_SCTP:
 507                 poff += sizeof(struct sctphdr);
 508                 break;
 509         }
 510
 511         return poff;
 512 }
 513
 514 /**
 515  * skb_get_poff - get the offset to the payload
 516  * @skb: sk_buff to get the payload offset from
 517  *
 518  * The function will get the offset to the payload as far as it could
 519  * be dissected.  The main user is currently BPF, so that we can dynamically
 520  * truncate packets without needing to push actual payload to the user
 521  * space and can analyze headers only, instead.
 522  */
 523 u32 skb_get_poff(const struct sk_buff *skb)
 524 {
 525         struct flow_keys keys;
 526
 527         if (!skb_flow_dissect_flow_keys(skb, &keys))
 528                 return 0;
 529
 530         return __skb_get_poff(skb, skb->data, &keys, skb_headlen(skb));
 531 }
 532
 533 static const struct flow_dissector_key flow_keys_dissector_keys[] = {
 534         {
 535                 .key_id = FLOW_DISSECTOR_KEY_BASIC,
 536                 .offset = offsetof(struct flow_keys, basic),
 537         },
 538         {
 539                 .key_id = FLOW_DISSECTOR_KEY_IPV4_ADDRS,
 540                 .offset = offsetof(struct flow_keys, addrs),
 541         },
 542         {
 543                 .key_id = FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS,
 544                 .offset = offsetof(struct flow_keys, addrs),
 545         },
 546         {
 547                 .key_id = FLOW_DISSECTOR_KEY_PORTS,
 548                 .offset = offsetof(struct flow_keys, ports),
 549         },
 550 };
 551
 552 static const struct flow_dissector_key flow_keys_buf_dissector_keys[] = {
 553         {
 554                 .key_id = FLOW_DISSECTOR_KEY_BASIC,
 555                 .offset = offsetof(struct flow_keys, basic),
 556         },
 557 };
 558
 559 struct flow_dissector flow_keys_dissector __read_mostly;
 560 EXPORT_SYMBOL(flow_keys_dissector);
 561
 562 struct flow_dissector flow_keys_buf_dissector __read_mostly;
 563
 564 static int __init init_default_flow_dissectors(void)
 565 {
 566         skb_flow_dissector_init(&flow_keys_dissector,
 567                                 flow_keys_dissector_keys,
 568                                 ARRAY_SIZE(flow_keys_dissector_keys));
 569         skb_flow_dissector_init(&flow_keys_buf_dissector,
 570                                 flow_keys_buf_dissector_keys,
 571                                 ARRAY_SIZE(flow_keys_buf_dissector_keys));
 572         return 0;
 573 }
 574
 575 late_initcall_sync(init_default_flow_dissectors);