net/core/flow_dissector.c

   1 #include <linux/kernel.h>
   2 #include <linux/skbuff.h>
   3 #include <linux/export.h>
   4 #include <linux/ip.h>
   5 #include <linux/ipv6.h>
   6 #include <linux/if_vlan.h>
   7 #include <net/ip.h>
   8 #include <net/ipv6.h>
   9 #include <linux/igmp.h>
  10 #include <linux/icmp.h>
  11 #include <linux/sctp.h>
  12 #include <linux/dccp.h>
  13 #include <linux/if_tunnel.h>
  14 #include <linux/if_pppox.h>
  15 #include <linux/ppp_defs.h>
  16 #include <linux/stddef.h>
  17 #include <net/flow_dissector.h>
  18 #include <scsi/fc/fc_fcoe.h>
  19
  20 static bool skb_flow_dissector_uses_key(struct flow_dissector *flow_dissector,
  21                                         enum flow_dissector_key_id key_id)
  22 {
  23         return flow_dissector->used_keys & (1 << key_id);
  24 }
  25
  26 static void skb_flow_dissector_set_key(struct flow_dissector *flow_dissector,
  27                                        enum flow_dissector_key_id key_id)
  28 {
  29         flow_dissector->used_keys |= (1 << key_id);
  30 }
  31
  32 static void *skb_flow_dissector_target(struct flow_dissector *flow_dissector,
  33                                        enum flow_dissector_key_id key_id,
  34                                        void *target_container)
  35 {
  36         return ((char *) target_container) + flow_dissector->offset[key_id];
  37 }
  38
  39 void skb_flow_dissector_init(struct flow_dissector *flow_dissector,
  40                              const struct flow_dissector_key *key,
  41                              unsigned int key_count)
  42 {
  43         unsigned int i;
  44
  45         memset(flow_dissector, 0, sizeof(*flow_dissector));
  46
  47         for (i = 0; i < key_count; i++, key++) {
  48                 /* User should make sure that every key target offset is withing
  49                  * boundaries of unsigned short.
  50                  */
  51                 BUG_ON(key->offset > USHRT_MAX);
  52                 BUG_ON(skb_flow_dissector_uses_key(flow_dissector,
  53                                                    key->key_id));
  54
  55                 skb_flow_dissector_set_key(flow_dissector, key->key_id);
  56                 flow_dissector->offset[key->key_id] = key->offset;
  57         }
  58
  59         /* Ensure that the dissector always includes basic key. That way
  60          * we are able to avoid handling lack of it in fast path.
  61          */
  62         BUG_ON(!skb_flow_dissector_uses_key(flow_dissector,
  63                                             FLOW_DISSECTOR_KEY_BASIC));
  64 }
  65 EXPORT_SYMBOL(skb_flow_dissector_init);
  66
  67 /**
  68  * __skb_flow_get_ports - extract the upper layer ports and return them
  69  * @skb: sk_buff to extract the ports from
  70  * @thoff: transport header offset
  71  * @ip_proto: protocol for which to get port offset
  72  * @data: raw buffer pointer to the packet, if NULL use skb->data
  73  * @hlen: packet header length, if @data is NULL use skb_headlen(skb)
  74  *
  75  * The function will try to retrieve the ports at offset thoff + poff where poff
  76  * is the protocol port offset returned from proto_ports_offset
  77  */
  78 __be32 __skb_flow_get_ports(const struct sk_buff *skb, int thoff, u8 ip_proto,
  79                             void *data, int hlen)
  80 {
  81         int poff = proto_ports_offset(ip_proto);
  82
  83         if (!data) {
  84                 data = skb->data;
  85                 hlen = skb_headlen(skb);
  86         }
  87
  88         if (poff >= 0) {
  89                 __be32 *ports, _ports;
  90
  91                 ports = __skb_header_pointer(skb, thoff + poff,
  92                                              sizeof(_ports), data, hlen, &_ports);
  93                 if (ports)
  94                         return *ports;
  95         }
  96
  97         return 0;
  98 }
  99 EXPORT_SYMBOL(__skb_flow_get_ports);
 100
 101 /**
 102  * __skb_flow_dissect - extract the flow_keys struct and return it
 103  * @skb: sk_buff to extract the flow from, can be NULL if the rest are specified
 104  * @flow_dissector: list of keys to dissect
 105  * @target_container: target structure to put dissected values into
 106  * @data: raw buffer pointer to the packet, if NULL use skb->data
 107  * @proto: protocol for which to get the flow, if @data is NULL use skb->protocol
 108  * @nhoff: network header offset, if @data is NULL use skb_network_offset(skb)
 109  * @hlen: packet header length, if @data is NULL use skb_headlen(skb)
 110  *
 111  * The function will try to retrieve individual keys into target specified
 112  * by flow_dissector from either the skbuff or a raw buffer specified by the
 113  * rest parameters.
 114  *
 115  * Caller must take care of zeroing target container memory.
 116  */
 117 bool __skb_flow_dissect(const struct sk_buff *skb,
 118                         struct flow_dissector *flow_dissector,
 119                         void *target_container,
 120                         void *data, __be16 proto, int nhoff, int hlen)
 121 {
 122         struct flow_dissector_key_basic *key_basic;
 123         struct flow_dissector_key_addrs *key_addrs;
 124         struct flow_dissector_key_ports *key_ports;
 125         u8 ip_proto;
 126
 127         if (!data) {
 128                 data = skb->data;
 129                 proto = skb->protocol;
 130                 nhoff = skb_network_offset(skb);
 131                 hlen = skb_headlen(skb);
 132         }
 133
 134         /* It is ensured by skb_flow_dissector_init() that basic key will
 135          * be always present.
 136          */
 137         key_basic = skb_flow_dissector_target(flow_dissector,
 138                                               FLOW_DISSECTOR_KEY_BASIC,
 139                                               target_container);
 140
 141 again:
 142         switch (proto) {
 143         case htons(ETH_P_IP): {
 144                 const struct iphdr *iph;
 145                 struct iphdr _iph;
 146 ip:
 147                 iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph);
 148                 if (!iph || iph->ihl < 5)
 149                         return false;
 150                 nhoff += iph->ihl * 4;
 151
 152                 ip_proto = iph->protocol;
 153                 if (ip_is_fragment(iph))
 154                         ip_proto = 0;
 155
 156                 if (!skb_flow_dissector_uses_key(flow_dissector,
 157                                                  FLOW_DISSECTOR_KEY_IPV4_ADDRS))
 158                         break;
 159                 key_addrs = skb_flow_dissector_target(flow_dissector,
 160                                                       FLOW_DISSECTOR_KEY_IPV4_ADDRS,
 161                                                       target_container);
 162                 memcpy(key_addrs, &iph->saddr, sizeof(*key_addrs));
 163                 break;
 164         }
 165         case htons(ETH_P_IPV6): {
 166                 const struct ipv6hdr *iph;
 167                 struct ipv6hdr _iph;
 168                 __be32 flow_label;
 169
 170 ipv6:
 171                 iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph);
 172                 if (!iph)
 173                         return false;
 174
 175                 ip_proto = iph->nexthdr;
 176                 nhoff += sizeof(struct ipv6hdr);
 177
 178                 if (!skb_flow_dissector_uses_key(flow_dissector,
 179                                                  FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS))
 180                         break;
 181                 key_addrs = skb_flow_dissector_target(flow_dissector,
 182                                                       FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS,
 183                                                       target_container);
 184
 185                 key_addrs->src = (__force __be32)ipv6_addr_hash(&iph->saddr);
 186                 key_addrs->dst = (__force __be32)ipv6_addr_hash(&iph->daddr);
 187
 188                 flow_label = ip6_flowlabel(iph);
 189                 if (flow_label) {
 190                         /* Awesome, IPv6 packet has a flow label so we can
 191                          * use that to represent the ports without any
 192                          * further dissection.
 193                          */
 194
 195                         key_basic->n_proto = proto;
 196                         key_basic->ip_proto = ip_proto;
 197                         key_basic->thoff = (u16)nhoff;
 198
 199                         if (!skb_flow_dissector_uses_key(flow_dissector,
 200                                                          FLOW_DISSECTOR_KEY_PORTS))
 201                                 break;
 202                         key_ports = skb_flow_dissector_target(flow_dissector,
 203                                                               FLOW_DISSECTOR_KEY_PORTS,
 204                                                               target_container);
 205                         key_ports->ports = flow_label;
 206
 207                         return true;
 208                 }
 209
 210                 break;
 211         }
 212         case htons(ETH_P_8021AD):
 213         case htons(ETH_P_8021Q): {
 214                 const struct vlan_hdr *vlan;
 215                 struct vlan_hdr _vlan;
 216
 217                 vlan = __skb_header_pointer(skb, nhoff, sizeof(_vlan), data, hlen, &_vlan);
 218                 if (!vlan)
 219                         return false;
 220
 221                 proto = vlan->h_vlan_encapsulated_proto;
 222                 nhoff += sizeof(*vlan);
 223                 goto again;
 224         }
 225         case htons(ETH_P_PPP_SES): {
 226                 struct {
 227                         struct pppoe_hdr hdr;
 228                         __be16 proto;
 229                 } *hdr, _hdr;
 230                 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
 231                 if (!hdr)
 232                         return false;
 233                 proto = hdr->proto;
 234                 nhoff += PPPOE_SES_HLEN;
 235                 switch (proto) {
 236                 case htons(PPP_IP):
 237                         goto ip;
 238                 case htons(PPP_IPV6):
 239                         goto ipv6;
 240                 default:
 241                         return false;
 242                 }
 243         }
 244         case htons(ETH_P_TIPC): {
 245                 struct {
 246                         __be32 pre[3];
 247                         __be32 srcnode;
 248                 } *hdr, _hdr;
 249                 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
 250                 if (!hdr)
 251                         return false;
 252                 key_basic->n_proto = proto;
 253                 key_basic->thoff = (u16)nhoff;
 254
 255                 if (skb_flow_dissector_uses_key(flow_dissector,
 256                                                 FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS)) {
 257                         return true;
 258                         key_addrs = skb_flow_dissector_target(flow_dissector,
 259                                                               FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS,
 260                                                               target_container);
 261                         key_addrs->src = hdr->srcnode;
 262                         key_addrs->dst = 0;
 263                 }
 264                 return true;
 265         }
 266         case htons(ETH_P_FCOE):
 267                 key_basic->thoff = (u16)(nhoff + FCOE_HEADER_LEN);
 268                 /* fall through */
 269         default:
 270                 return false;
 271         }
 272
 273         switch (ip_proto) {
 274         case IPPROTO_GRE: {
 275                 struct gre_hdr {
 276                         __be16 flags;
 277                         __be16 proto;
 278                 } *hdr, _hdr;
 279
 280                 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
 281                 if (!hdr)
 282                         return false;
 283                 /*
 284                  * Only look inside GRE if version zero and no
 285                  * routing
 286                  */
 287                 if (!(hdr->flags & (GRE_VERSION|GRE_ROUTING))) {
 288                         proto = hdr->proto;
 289                         nhoff += 4;
 290                         if (hdr->flags & GRE_CSUM)
 291                                 nhoff += 4;
 292                         if (hdr->flags & GRE_KEY)
 293                                 nhoff += 4;
 294                         if (hdr->flags & GRE_SEQ)
 295                                 nhoff += 4;
 296                         if (proto == htons(ETH_P_TEB)) {
 297                                 const struct ethhdr *eth;
 298                                 struct ethhdr _eth;
 299
 300                                 eth = __skb_header_pointer(skb, nhoff,
 301                                                            sizeof(_eth),
 302                                                            data, hlen, &_eth);
 303                                 if (!eth)
 304                                         return false;
 305                                 proto = eth->h_proto;
 306                                 nhoff += sizeof(*eth);
 307                         }
 308                         goto again;
 309                 }
 310                 break;
 311         }
 312         case IPPROTO_IPIP:
 313                 proto = htons(ETH_P_IP);
 314                 goto ip;
 315         case IPPROTO_IPV6:
 316                 proto = htons(ETH_P_IPV6);
 317                 goto ipv6;
 318         default:
 319                 break;
 320         }
 321
 322         /* It is ensured by skb_flow_dissector_init() that basic key will
 323          * be always present.
 324          */
 325         key_basic = skb_flow_dissector_target(flow_dissector,
 326                                               FLOW_DISSECTOR_KEY_BASIC,
 327                                               target_container);
 328         key_basic->n_proto = proto;
 329         key_basic->ip_proto = ip_proto;
 330         key_basic->thoff = (u16) nhoff;
 331
 332         if (skb_flow_dissector_uses_key(flow_dissector,
 333                                         FLOW_DISSECTOR_KEY_PORTS)) {
 334                 key_ports = skb_flow_dissector_target(flow_dissector,
 335                                                       FLOW_DISSECTOR_KEY_PORTS,
 336                                                       target_container);
 337                 key_ports->ports = __skb_flow_get_ports(skb, nhoff, ip_proto,
 338                                                         data, hlen);
 339         }
 340
 341         return true;
 342 }
 343 EXPORT_SYMBOL(__skb_flow_dissect);
 344
 345 static u32 hashrnd __read_mostly;
 346 static __always_inline void __flow_hash_secret_init(void)
 347 {
 348         net_get_random_once(&hashrnd, sizeof(hashrnd));
 349 }
 350
 351 static __always_inline u32 __flow_hash_3words(u32 a, u32 b, u32 c, u32 keyval)
 352 {
 353         return jhash_3words(a, b, c, keyval);
 354 }
 355
 356 static inline u32 __flow_hash_from_keys(struct flow_keys *keys, u32 keyval)
 357 {
 358         u32 hash;
 359
 360         /* get a consistent hash (same value on both flow directions) */
 361         if (((__force u32)keys->addrs.dst < (__force u32)keys->addrs.src) ||
 362             (((__force u32)keys->addrs.dst == (__force u32)keys->addrs.src) &&
 363              ((__force u16)keys->ports.port16[1] < (__force u16)keys->ports.port16[0]))) {
 364                 swap(keys->addrs.dst, keys->addrs.src);
 365                 swap(keys->ports.port16[0], keys->ports.port16[1]);
 366         }
 367
 368         hash = __flow_hash_3words((__force u32)keys->addrs.dst,
 369                                   (__force u32)keys->addrs.src,
 370                                   (__force u32)keys->ports.ports,
 371                                   keyval);
 372         if (!hash)
 373                 hash = 1;
 374
 375         return hash;
 376 }
 377
 378 u32 flow_hash_from_keys(struct flow_keys *keys)
 379 {
 380         __flow_hash_secret_init();
 381         return __flow_hash_from_keys(keys, hashrnd);
 382 }
 383 EXPORT_SYMBOL(flow_hash_from_keys);
 384
 385 static inline u32 ___skb_get_hash(const struct sk_buff *skb,
 386                                   struct flow_keys *keys, u32 keyval)
 387 {
 388         if (!skb_flow_dissect_flow_keys(skb, keys))
 389                 return 0;
 390
 391         return __flow_hash_from_keys(keys, keyval);
 392 }
 393
 394 struct _flow_keys_digest_data {
 395         __be16  n_proto;
 396         u8      ip_proto;
 397         u8      padding;
 398         __be32  ports;
 399         __be32  src;
 400         __be32  dst;
 401 };
 402
 403 void make_flow_keys_digest(struct flow_keys_digest *digest,
 404                            const struct flow_keys *flow)
 405 {
 406         struct _flow_keys_digest_data *data =
 407             (struct _flow_keys_digest_data *)digest;
 408
 409         BUILD_BUG_ON(sizeof(*data) > sizeof(*digest));
 410
 411         memset(digest, 0, sizeof(*digest));
 412
 413         data->n_proto = flow->basic.n_proto;
 414         data->ip_proto = flow->basic.ip_proto;
 415         data->ports = flow->ports.ports;
 416         data->src = flow->addrs.src;
 417         data->dst = flow->addrs.dst;
 418 }
 419 EXPORT_SYMBOL(make_flow_keys_digest);
 420
 421 /**
 422  * __skb_get_hash: calculate a flow hash
 423  * @skb: sk_buff to calculate flow hash from
 424  *
 425  * This function calculates a flow hash based on src/dst addresses
 426  * and src/dst port numbers.  Sets hash in skb to non-zero hash value
 427  * on success, zero indicates no valid hash.  Also, sets l4_hash in skb
 428  * if hash is a canonical 4-tuple hash over transport ports.
 429  */
 430 void __skb_get_hash(struct sk_buff *skb)
 431 {
 432         struct flow_keys keys;
 433         u32 hash;
 434
 435         __flow_hash_secret_init();
 436
 437         hash = ___skb_get_hash(skb, &keys, hashrnd);
 438         if (!hash)
 439                 return;
 440         if (keys.ports.ports)
 441                 skb->l4_hash = 1;
 442         skb->sw_hash = 1;
 443         skb->hash = hash;
 444 }
 445 EXPORT_SYMBOL(__skb_get_hash);
 446
 447 __u32 skb_get_hash_perturb(const struct sk_buff *skb, u32 perturb)
 448 {
 449         struct flow_keys keys;
 450
 451         return ___skb_get_hash(skb, &keys, perturb);
 452 }
 453 EXPORT_SYMBOL(skb_get_hash_perturb);
 454
 455 u32 __skb_get_poff(const struct sk_buff *skb, void *data,
 456                    const struct flow_keys *keys, int hlen)
 457 {
 458         u32 poff = keys->basic.thoff;
 459
 460         switch (keys->basic.ip_proto) {
 461         case IPPROTO_TCP: {
 462                 /* access doff as u8 to avoid unaligned access */
 463                 const u8 *doff;
 464                 u8 _doff;
 465
 466                 doff = __skb_header_pointer(skb, poff + 12, sizeof(_doff),
 467                                             data, hlen, &_doff);
 468                 if (!doff)
 469                         return poff;
 470
 471                 poff += max_t(u32, sizeof(struct tcphdr), (*doff & 0xF0) >> 2);
 472                 break;
 473         }
 474         case IPPROTO_UDP:
 475         case IPPROTO_UDPLITE:
 476                 poff += sizeof(struct udphdr);
 477                 break;
 478         /* For the rest, we do not really care about header
 479          * extensions at this point for now.
 480          */
 481         case IPPROTO_ICMP:
 482                 poff += sizeof(struct icmphdr);
 483                 break;
 484         case IPPROTO_ICMPV6:
 485                 poff += sizeof(struct icmp6hdr);
 486                 break;
 487         case IPPROTO_IGMP:
 488                 poff += sizeof(struct igmphdr);
 489                 break;
 490         case IPPROTO_DCCP:
 491                 poff += sizeof(struct dccp_hdr);
 492                 break;
 493         case IPPROTO_SCTP:
 494                 poff += sizeof(struct sctphdr);
 495                 break;
 496         }
 497
 498         return poff;
 499 }
 500
 501 /**
 502  * skb_get_poff - get the offset to the payload
 503  * @skb: sk_buff to get the payload offset from
 504  *
 505  * The function will get the offset to the payload as far as it could
 506  * be dissected.  The main user is currently BPF, so that we can dynamically
 507  * truncate packets without needing to push actual payload to the user
 508  * space and can analyze headers only, instead.
 509  */
 510 u32 skb_get_poff(const struct sk_buff *skb)
 511 {
 512         struct flow_keys keys;
 513
 514         if (!skb_flow_dissect_flow_keys(skb, &keys))
 515                 return 0;
 516
 517         return __skb_get_poff(skb, skb->data, &keys, skb_headlen(skb));
 518 }
 519
 520 static const struct flow_dissector_key flow_keys_dissector_keys[] = {
 521         {
 522                 .key_id = FLOW_DISSECTOR_KEY_BASIC,
 523                 .offset = offsetof(struct flow_keys, basic),
 524         },
 525         {
 526                 .key_id = FLOW_DISSECTOR_KEY_IPV4_ADDRS,
 527                 .offset = offsetof(struct flow_keys, addrs),
 528         },
 529         {
 530                 .key_id = FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS,
 531                 .offset = offsetof(struct flow_keys, addrs),
 532         },
 533         {
 534                 .key_id = FLOW_DISSECTOR_KEY_PORTS,
 535                 .offset = offsetof(struct flow_keys, ports),
 536         },
 537 };
 538
 539 static const struct flow_dissector_key flow_keys_buf_dissector_keys[] = {
 540         {
 541                 .key_id = FLOW_DISSECTOR_KEY_BASIC,
 542                 .offset = offsetof(struct flow_keys, basic),
 543         },
 544 };
 545
 546 struct flow_dissector flow_keys_dissector __read_mostly;
 547 EXPORT_SYMBOL(flow_keys_dissector);
 548
 549 struct flow_dissector flow_keys_buf_dissector __read_mostly;
 550
 551 static int __init init_default_flow_dissectors(void)
 552 {
 553         skb_flow_dissector_init(&flow_keys_dissector,
 554                                 flow_keys_dissector_keys,
 555                                 ARRAY_SIZE(flow_keys_dissector_keys));
 556         skb_flow_dissector_init(&flow_keys_buf_dissector,
 557                                 flow_keys_buf_dissector_keys,
 558                                 ARRAY_SIZE(flow_keys_buf_dissector_keys));
 559         return 0;
 560 }
 561
 562 late_initcall_sync(init_default_flow_dissectors);