net/core/flow_dissector.c

   1 #include <linux/kernel.h>
   2 #include <linux/skbuff.h>
   3 #include <linux/export.h>
   4 #include <linux/ip.h>
   5 #include <linux/ipv6.h>
   6 #include <linux/if_vlan.h>
   7 #include <net/ip.h>
   8 #include <net/ipv6.h>
   9 #include <linux/igmp.h>
  10 #include <linux/icmp.h>
  11 #include <linux/sctp.h>
  12 #include <linux/dccp.h>
  13 #include <linux/if_tunnel.h>
  14 #include <linux/if_pppox.h>
  15 #include <linux/ppp_defs.h>
  16 #include <linux/stddef.h>
  17 #include <linux/if_ether.h>
  18 #include <net/flow_dissector.h>
  19 #include <scsi/fc/fc_fcoe.h>
  20
  21 static bool skb_flow_dissector_uses_key(struct flow_dissector *flow_dissector,
  22                                         enum flow_dissector_key_id key_id)
  23 {
  24         return flow_dissector->used_keys & (1 << key_id);
  25 }
  26
  27 static void skb_flow_dissector_set_key(struct flow_dissector *flow_dissector,
  28                                        enum flow_dissector_key_id key_id)
  29 {
  30         flow_dissector->used_keys |= (1 << key_id);
  31 }
  32
  33 static void *skb_flow_dissector_target(struct flow_dissector *flow_dissector,
  34                                        enum flow_dissector_key_id key_id,
  35                                        void *target_container)
  36 {
  37         return ((char *) target_container) + flow_dissector->offset[key_id];
  38 }
  39
  40 void skb_flow_dissector_init(struct flow_dissector *flow_dissector,
  41                              const struct flow_dissector_key *key,
  42                              unsigned int key_count)
  43 {
  44         unsigned int i;
  45
  46         memset(flow_dissector, 0, sizeof(*flow_dissector));
  47
  48         for (i = 0; i < key_count; i++, key++) {
  49                 /* User should make sure that every key target offset is withing
  50                  * boundaries of unsigned short.
  51                  */
  52                 BUG_ON(key->offset > USHRT_MAX);
  53                 BUG_ON(skb_flow_dissector_uses_key(flow_dissector,
  54                                                    key->key_id));
  55
  56                 skb_flow_dissector_set_key(flow_dissector, key->key_id);
  57                 flow_dissector->offset[key->key_id] = key->offset;
  58         }
  59
  60         /* Ensure that the dissector always includes basic key. That way
  61          * we are able to avoid handling lack of it in fast path.
  62          */
  63         BUG_ON(!skb_flow_dissector_uses_key(flow_dissector,
  64                                             FLOW_DISSECTOR_KEY_BASIC));
  65 }
  66 EXPORT_SYMBOL(skb_flow_dissector_init);
  67
  68 /**
  69  * __skb_flow_get_ports - extract the upper layer ports and return them
  70  * @skb: sk_buff to extract the ports from
  71  * @thoff: transport header offset
  72  * @ip_proto: protocol for which to get port offset
  73  * @data: raw buffer pointer to the packet, if NULL use skb->data
  74  * @hlen: packet header length, if @data is NULL use skb_headlen(skb)
  75  *
  76  * The function will try to retrieve the ports at offset thoff + poff where poff
  77  * is the protocol port offset returned from proto_ports_offset
  78  */
  79 __be32 __skb_flow_get_ports(const struct sk_buff *skb, int thoff, u8 ip_proto,
  80                             void *data, int hlen)
  81 {
  82         int poff = proto_ports_offset(ip_proto);
  83
  84         if (!data) {
  85                 data = skb->data;
  86                 hlen = skb_headlen(skb);
  87         }
  88
  89         if (poff >= 0) {
  90                 __be32 *ports, _ports;
  91
  92                 ports = __skb_header_pointer(skb, thoff + poff,
  93                                              sizeof(_ports), data, hlen, &_ports);
  94                 if (ports)
  95                         return *ports;
  96         }
  97
  98         return 0;
  99 }
 100 EXPORT_SYMBOL(__skb_flow_get_ports);
 101
 102 /**
 103  * __skb_flow_dissect - extract the flow_keys struct and return it
 104  * @skb: sk_buff to extract the flow from, can be NULL if the rest are specified
 105  * @flow_dissector: list of keys to dissect
 106  * @target_container: target structure to put dissected values into
 107  * @data: raw buffer pointer to the packet, if NULL use skb->data
 108  * @proto: protocol for which to get the flow, if @data is NULL use skb->protocol
 109  * @nhoff: network header offset, if @data is NULL use skb_network_offset(skb)
 110  * @hlen: packet header length, if @data is NULL use skb_headlen(skb)
 111  *
 112  * The function will try to retrieve individual keys into target specified
 113  * by flow_dissector from either the skbuff or a raw buffer specified by the
 114  * rest parameters.
 115  *
 116  * Caller must take care of zeroing target container memory.
 117  */
 118 bool __skb_flow_dissect(const struct sk_buff *skb,
 119                         struct flow_dissector *flow_dissector,
 120                         void *target_container,
 121                         void *data, __be16 proto, int nhoff, int hlen)
 122 {
 123         struct flow_dissector_key_basic *key_basic;
 124         struct flow_dissector_key_addrs *key_addrs;
 125         struct flow_dissector_key_ports *key_ports;
 126         u8 ip_proto;
 127
 128         if (!data) {
 129                 data = skb->data;
 130                 proto = skb->protocol;
 131                 nhoff = skb_network_offset(skb);
 132                 hlen = skb_headlen(skb);
 133         }
 134
 135         /* It is ensured by skb_flow_dissector_init() that basic key will
 136          * be always present.
 137          */
 138         key_basic = skb_flow_dissector_target(flow_dissector,
 139                                               FLOW_DISSECTOR_KEY_BASIC,
 140                                               target_container);
 141
 142         if (skb_flow_dissector_uses_key(flow_dissector,
 143                                         FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
 144                 struct ethhdr *eth = eth_hdr(skb);
 145                 struct flow_dissector_key_eth_addrs *key_eth_addrs;
 146
 147                 key_eth_addrs = skb_flow_dissector_target(flow_dissector,
 148                                                           FLOW_DISSECTOR_KEY_ETH_ADDRS,
 149                                                           target_container);
 150                 memcpy(key_eth_addrs, &eth->h_dest, sizeof(*key_eth_addrs));
 151         }
 152
 153 again:
 154         switch (proto) {
 155         case htons(ETH_P_IP): {
 156                 const struct iphdr *iph;
 157                 struct iphdr _iph;
 158 ip:
 159                 iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph);
 160                 if (!iph || iph->ihl < 5)
 161                         return false;
 162                 nhoff += iph->ihl * 4;
 163
 164                 ip_proto = iph->protocol;
 165                 if (ip_is_fragment(iph))
 166                         ip_proto = 0;
 167
 168                 if (!skb_flow_dissector_uses_key(flow_dissector,
 169                                                  FLOW_DISSECTOR_KEY_IPV4_ADDRS))
 170                         break;
 171                 key_addrs = skb_flow_dissector_target(flow_dissector,
 172                                                       FLOW_DISSECTOR_KEY_IPV4_ADDRS,
 173                                                       target_container);
 174                 memcpy(key_addrs, &iph->saddr, sizeof(*key_addrs));
 175                 break;
 176         }
 177         case htons(ETH_P_IPV6): {
 178                 const struct ipv6hdr *iph;
 179                 struct ipv6hdr _iph;
 180                 __be32 flow_label;
 181
 182 ipv6:
 183                 iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph);
 184                 if (!iph)
 185                         return false;
 186
 187                 ip_proto = iph->nexthdr;
 188                 nhoff += sizeof(struct ipv6hdr);
 189
 190                 if (skb_flow_dissector_uses_key(flow_dissector,
 191                                                 FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS)) {
 192                         key_addrs = skb_flow_dissector_target(flow_dissector,
 193                                                               FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS,
 194                                                               target_container);
 195
 196                         key_addrs->src = (__force __be32)ipv6_addr_hash(&iph->saddr);
 197                         key_addrs->dst = (__force __be32)ipv6_addr_hash(&iph->daddr);
 198                         goto flow_label;
 199                 }
 200                 if (skb_flow_dissector_uses_key(flow_dissector,
 201                                                 FLOW_DISSECTOR_KEY_IPV6_ADDRS)) {
 202                         struct flow_dissector_key_ipv6_addrs *key_ipv6_addrs;
 203
 204                         key_ipv6_addrs = skb_flow_dissector_target(flow_dissector,
 205                                                                    FLOW_DISSECTOR_KEY_IPV6_ADDRS,
 206                                                                    target_container);
 207
 208                         memcpy(key_ipv6_addrs, &iph->saddr, sizeof(*key_ipv6_addrs));
 209                         goto flow_label;
 210                 }
 211                 break;
 212 flow_label:
 213                 flow_label = ip6_flowlabel(iph);
 214                 if (flow_label) {
 215                         /* Awesome, IPv6 packet has a flow label so we can
 216                          * use that to represent the ports without any
 217                          * further dissection.
 218                          */
 219
 220                         key_basic->n_proto = proto;
 221                         key_basic->ip_proto = ip_proto;
 222                         key_basic->thoff = (u16)nhoff;
 223
 224                         if (!skb_flow_dissector_uses_key(flow_dissector,
 225                                                          FLOW_DISSECTOR_KEY_PORTS))
 226                                 break;
 227                         key_ports = skb_flow_dissector_target(flow_dissector,
 228                                                               FLOW_DISSECTOR_KEY_PORTS,
 229                                                               target_container);
 230                         key_ports->ports = flow_label;
 231
 232                         return true;
 233                 }
 234
 235                 break;
 236         }
 237         case htons(ETH_P_8021AD):
 238         case htons(ETH_P_8021Q): {
 239                 const struct vlan_hdr *vlan;
 240                 struct vlan_hdr _vlan;
 241
 242                 vlan = __skb_header_pointer(skb, nhoff, sizeof(_vlan), data, hlen, &_vlan);
 243                 if (!vlan)
 244                         return false;
 245
 246                 proto = vlan->h_vlan_encapsulated_proto;
 247                 nhoff += sizeof(*vlan);
 248                 goto again;
 249         }
 250         case htons(ETH_P_PPP_SES): {
 251                 struct {
 252                         struct pppoe_hdr hdr;
 253                         __be16 proto;
 254                 } *hdr, _hdr;
 255                 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
 256                 if (!hdr)
 257                         return false;
 258                 proto = hdr->proto;
 259                 nhoff += PPPOE_SES_HLEN;
 260                 switch (proto) {
 261                 case htons(PPP_IP):
 262                         goto ip;
 263                 case htons(PPP_IPV6):
 264                         goto ipv6;
 265                 default:
 266                         return false;
 267                 }
 268         }
 269         case htons(ETH_P_TIPC): {
 270                 struct {
 271                         __be32 pre[3];
 272                         __be32 srcnode;
 273                 } *hdr, _hdr;
 274                 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
 275                 if (!hdr)
 276                         return false;
 277                 key_basic->n_proto = proto;
 278                 key_basic->thoff = (u16)nhoff;
 279
 280                 if (skb_flow_dissector_uses_key(flow_dissector,
 281                                                 FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS)) {
 282                         key_addrs = skb_flow_dissector_target(flow_dissector,
 283                                                               FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS,
 284                                                               target_container);
 285                         key_addrs->src = hdr->srcnode;
 286                         key_addrs->dst = 0;
 287                 }
 288                 return true;
 289         }
 290         case htons(ETH_P_FCOE):
 291                 key_basic->thoff = (u16)(nhoff + FCOE_HEADER_LEN);
 292                 /* fall through */
 293         default:
 294                 return false;
 295         }
 296
 297         switch (ip_proto) {
 298         case IPPROTO_GRE: {
 299                 struct gre_hdr {
 300                         __be16 flags;
 301                         __be16 proto;
 302                 } *hdr, _hdr;
 303
 304                 hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
 305                 if (!hdr)
 306                         return false;
 307                 /*
 308                  * Only look inside GRE if version zero and no
 309                  * routing
 310                  */
 311                 if (!(hdr->flags & (GRE_VERSION|GRE_ROUTING))) {
 312                         proto = hdr->proto;
 313                         nhoff += 4;
 314                         if (hdr->flags & GRE_CSUM)
 315                                 nhoff += 4;
 316                         if (hdr->flags & GRE_KEY)
 317                                 nhoff += 4;
 318                         if (hdr->flags & GRE_SEQ)
 319                                 nhoff += 4;
 320                         if (proto == htons(ETH_P_TEB)) {
 321                                 const struct ethhdr *eth;
 322                                 struct ethhdr _eth;
 323
 324                                 eth = __skb_header_pointer(skb, nhoff,
 325                                                            sizeof(_eth),
 326                                                            data, hlen, &_eth);
 327                                 if (!eth)
 328                                         return false;
 329                                 proto = eth->h_proto;
 330                                 nhoff += sizeof(*eth);
 331                         }
 332                         goto again;
 333                 }
 334                 break;
 335         }
 336         case IPPROTO_IPIP:
 337                 proto = htons(ETH_P_IP);
 338                 goto ip;
 339         case IPPROTO_IPV6:
 340                 proto = htons(ETH_P_IPV6);
 341                 goto ipv6;
 342         default:
 343                 break;
 344         }
 345
 346         /* It is ensured by skb_flow_dissector_init() that basic key will
 347          * be always present.
 348          */
 349         key_basic = skb_flow_dissector_target(flow_dissector,
 350                                               FLOW_DISSECTOR_KEY_BASIC,
 351                                               target_container);
 352         key_basic->n_proto = proto;
 353         key_basic->ip_proto = ip_proto;
 354         key_basic->thoff = (u16) nhoff;
 355
 356         if (skb_flow_dissector_uses_key(flow_dissector,
 357                                         FLOW_DISSECTOR_KEY_PORTS)) {
 358                 key_ports = skb_flow_dissector_target(flow_dissector,
 359                                                       FLOW_DISSECTOR_KEY_PORTS,
 360                                                       target_container);
 361                 key_ports->ports = __skb_flow_get_ports(skb, nhoff, ip_proto,
 362                                                         data, hlen);
 363         }
 364
 365         return true;
 366 }
 367 EXPORT_SYMBOL(__skb_flow_dissect);
 368
 369 static u32 hashrnd __read_mostly;
 370 static __always_inline void __flow_hash_secret_init(void)
 371 {
 372         net_get_random_once(&hashrnd, sizeof(hashrnd));
 373 }
 374
 375 static __always_inline u32 __flow_hash_3words(u32 a, u32 b, u32 c, u32 keyval)
 376 {
 377         return jhash_3words(a, b, c, keyval);
 378 }
 379
 380 static inline u32 __flow_hash_from_keys(struct flow_keys *keys, u32 keyval)
 381 {
 382         u32 hash;
 383
 384         /* get a consistent hash (same value on both flow directions) */
 385         if (((__force u32)keys->addrs.dst < (__force u32)keys->addrs.src) ||
 386             (((__force u32)keys->addrs.dst == (__force u32)keys->addrs.src) &&
 387              ((__force u16)keys->ports.dst < (__force u16)keys->ports.src))) {
 388                 swap(keys->addrs.dst, keys->addrs.src);
 389                 swap(keys->ports.src, keys->ports.dst);
 390         }
 391
 392         hash = __flow_hash_3words((__force u32)keys->addrs.dst,
 393                                   (__force u32)keys->addrs.src,
 394                                   (__force u32)keys->ports.ports,
 395                                   keyval);
 396         if (!hash)
 397                 hash = 1;
 398
 399         return hash;
 400 }
 401
 402 u32 flow_hash_from_keys(struct flow_keys *keys)
 403 {
 404         __flow_hash_secret_init();
 405         return __flow_hash_from_keys(keys, hashrnd);
 406 }
 407 EXPORT_SYMBOL(flow_hash_from_keys);
 408
 409 static inline u32 ___skb_get_hash(const struct sk_buff *skb,
 410                                   struct flow_keys *keys, u32 keyval)
 411 {
 412         if (!skb_flow_dissect_flow_keys(skb, keys))
 413                 return 0;
 414
 415         return __flow_hash_from_keys(keys, keyval);
 416 }
 417
 418 struct _flow_keys_digest_data {
 419         __be16  n_proto;
 420         u8      ip_proto;
 421         u8      padding;
 422         __be32  ports;
 423         __be32  src;
 424         __be32  dst;
 425 };
 426
 427 void make_flow_keys_digest(struct flow_keys_digest *digest,
 428                            const struct flow_keys *flow)
 429 {
 430         struct _flow_keys_digest_data *data =
 431             (struct _flow_keys_digest_data *)digest;
 432
 433         BUILD_BUG_ON(sizeof(*data) > sizeof(*digest));
 434
 435         memset(digest, 0, sizeof(*digest));
 436
 437         data->n_proto = flow->basic.n_proto;
 438         data->ip_proto = flow->basic.ip_proto;
 439         data->ports = flow->ports.ports;
 440         data->src = flow->addrs.src;
 441         data->dst = flow->addrs.dst;
 442 }
 443 EXPORT_SYMBOL(make_flow_keys_digest);
 444
 445 /**
 446  * __skb_get_hash: calculate a flow hash
 447  * @skb: sk_buff to calculate flow hash from
 448  *
 449  * This function calculates a flow hash based on src/dst addresses
 450  * and src/dst port numbers.  Sets hash in skb to non-zero hash value
 451  * on success, zero indicates no valid hash.  Also, sets l4_hash in skb
 452  * if hash is a canonical 4-tuple hash over transport ports.
 453  */
 454 void __skb_get_hash(struct sk_buff *skb)
 455 {
 456         struct flow_keys keys;
 457         u32 hash;
 458
 459         __flow_hash_secret_init();
 460
 461         hash = ___skb_get_hash(skb, &keys, hashrnd);
 462         if (!hash)
 463                 return;
 464         if (keys.ports.ports)
 465                 skb->l4_hash = 1;
 466         skb->sw_hash = 1;
 467         skb->hash = hash;
 468 }
 469 EXPORT_SYMBOL(__skb_get_hash);
 470
 471 __u32 skb_get_hash_perturb(const struct sk_buff *skb, u32 perturb)
 472 {
 473         struct flow_keys keys;
 474
 475         return ___skb_get_hash(skb, &keys, perturb);
 476 }
 477 EXPORT_SYMBOL(skb_get_hash_perturb);
 478
 479 u32 __skb_get_poff(const struct sk_buff *skb, void *data,
 480                    const struct flow_keys *keys, int hlen)
 481 {
 482         u32 poff = keys->basic.thoff;
 483
 484         switch (keys->basic.ip_proto) {
 485         case IPPROTO_TCP: {
 486                 /* access doff as u8 to avoid unaligned access */
 487                 const u8 *doff;
 488                 u8 _doff;
 489
 490                 doff = __skb_header_pointer(skb, poff + 12, sizeof(_doff),
 491                                             data, hlen, &_doff);
 492                 if (!doff)
 493                         return poff;
 494
 495                 poff += max_t(u32, sizeof(struct tcphdr), (*doff & 0xF0) >> 2);
 496                 break;
 497         }
 498         case IPPROTO_UDP:
 499         case IPPROTO_UDPLITE:
 500                 poff += sizeof(struct udphdr);
 501                 break;
 502         /* For the rest, we do not really care about header
 503          * extensions at this point for now.
 504          */
 505         case IPPROTO_ICMP:
 506                 poff += sizeof(struct icmphdr);
 507                 break;
 508         case IPPROTO_ICMPV6:
 509                 poff += sizeof(struct icmp6hdr);
 510                 break;
 511         case IPPROTO_IGMP:
 512                 poff += sizeof(struct igmphdr);
 513                 break;
 514         case IPPROTO_DCCP:
 515                 poff += sizeof(struct dccp_hdr);
 516                 break;
 517         case IPPROTO_SCTP:
 518                 poff += sizeof(struct sctphdr);
 519                 break;
 520         }
 521
 522         return poff;
 523 }
 524
 525 /**
 526  * skb_get_poff - get the offset to the payload
 527  * @skb: sk_buff to get the payload offset from
 528  *
 529  * The function will get the offset to the payload as far as it could
 530  * be dissected.  The main user is currently BPF, so that we can dynamically
 531  * truncate packets without needing to push actual payload to the user
 532  * space and can analyze headers only, instead.
 533  */
 534 u32 skb_get_poff(const struct sk_buff *skb)
 535 {
 536         struct flow_keys keys;
 537
 538         if (!skb_flow_dissect_flow_keys(skb, &keys))
 539                 return 0;
 540
 541         return __skb_get_poff(skb, skb->data, &keys, skb_headlen(skb));
 542 }
 543
 544 static const struct flow_dissector_key flow_keys_dissector_keys[] = {
 545         {
 546                 .key_id = FLOW_DISSECTOR_KEY_BASIC,
 547                 .offset = offsetof(struct flow_keys, basic),
 548         },
 549         {
 550                 .key_id = FLOW_DISSECTOR_KEY_IPV4_ADDRS,
 551                 .offset = offsetof(struct flow_keys, addrs),
 552         },
 553         {
 554                 .key_id = FLOW_DISSECTOR_KEY_IPV6_HASH_ADDRS,
 555                 .offset = offsetof(struct flow_keys, addrs),
 556         },
 557         {
 558                 .key_id = FLOW_DISSECTOR_KEY_PORTS,
 559                 .offset = offsetof(struct flow_keys, ports),
 560         },
 561 };
 562
 563 static const struct flow_dissector_key flow_keys_buf_dissector_keys[] = {
 564         {
 565                 .key_id = FLOW_DISSECTOR_KEY_BASIC,
 566                 .offset = offsetof(struct flow_keys, basic),
 567         },
 568 };
 569
 570 struct flow_dissector flow_keys_dissector __read_mostly;
 571 EXPORT_SYMBOL(flow_keys_dissector);
 572
 573 struct flow_dissector flow_keys_buf_dissector __read_mostly;
 574
 575 static int __init init_default_flow_dissectors(void)
 576 {
 577         skb_flow_dissector_init(&flow_keys_dissector,
 578                                 flow_keys_dissector_keys,
 579                                 ARRAY_SIZE(flow_keys_dissector_keys));
 580         skb_flow_dissector_init(&flow_keys_buf_dissector,
 581                                 flow_keys_buf_dissector_keys,
 582                                 ARRAY_SIZE(flow_keys_buf_dissector_keys));
 583         return 0;
 584 }
 585
 586 late_initcall_sync(init_default_flow_dissectors);