drivers/net/ethernet/sfc/filter.c

   1 /****************************************************************************
   2  * Driver for Solarflare Solarstorm network controllers and boards
   3  * Copyright 2005-2010 Solarflare Communications Inc.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 as published
   7  * by the Free Software Foundation, incorporated herein by reference.
   8  */
   9
  10 #include <linux/in.h>
  11 #include <net/ip.h>
  12 #include "efx.h"
  13 #include "filter.h"
  14 #include "io.h"
  15 #include "nic.h"
  16 #include "regs.h"
  17
  18 /* "Fudge factors" - difference between programmed value and actual depth.
  19  * Due to pipelined implementation we need to program H/W with a value that
  20  * is larger than the hop limit we want.
  21  */
  22 #define FILTER_CTL_SRCH_FUDGE_WILD 3
  23 #define FILTER_CTL_SRCH_FUDGE_FULL 1
  24
  25 /* Hard maximum hop limit.  Hardware will time-out beyond 200-something.
  26  * We also need to avoid infinite loops in efx_filter_search() when the
  27  * table is full.
  28  */
  29 #define FILTER_CTL_SRCH_MAX 200
  30
  31 /* Don't try very hard to find space for performance hints, as this is
  32  * counter-productive. */
  33 #define FILTER_CTL_SRCH_HINT_MAX 5
  34
  35 enum efx_filter_table_id {
  36         EFX_FILTER_TABLE_RX_IP = 0,
  37         EFX_FILTER_TABLE_RX_MAC,
  38         EFX_FILTER_TABLE_RX_DEF,
  39         EFX_FILTER_TABLE_TX_MAC,
  40         EFX_FILTER_TABLE_COUNT,
  41 };
  42
  43 enum efx_filter_index {
  44         EFX_FILTER_INDEX_UC_DEF,
  45         EFX_FILTER_INDEX_MC_DEF,
  46         EFX_FILTER_SIZE_RX_DEF,
  47 };
  48
  49 struct efx_filter_table {
  50         enum efx_filter_table_id id;
  51         u32             offset;         /* address of table relative to BAR */
  52         unsigned        size;           /* number of entries */
  53         unsigned        step;           /* step between entries */
  54         unsigned        used;           /* number currently used */
  55         unsigned long   *used_bitmap;
  56         struct efx_filter_spec *spec;
  57         unsigned        search_depth[EFX_FILTER_TYPE_COUNT];
  58 };
  59
  60 struct efx_filter_state {
  61         spinlock_t      lock;
  62         struct efx_filter_table table[EFX_FILTER_TABLE_COUNT];
  63 #ifdef CONFIG_RFS_ACCEL
  64         u32             *rps_flow_id;
  65         unsigned        rps_expire_index;
  66 #endif
  67 };
  68
  69 /* The filter hash function is LFSR polynomial x^16 + x^3 + 1 of a 32-bit
  70  * key derived from the n-tuple.  The initial LFSR state is 0xffff. */
  71 static u16 efx_filter_hash(u32 key)
  72 {
  73         u16 tmp;
  74
  75         /* First 16 rounds */
  76         tmp = 0x1fff ^ key >> 16;
  77         tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
  78         tmp = tmp ^ tmp >> 9;
  79         /* Last 16 rounds */
  80         tmp = tmp ^ tmp << 13 ^ key;
  81         tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
  82         return tmp ^ tmp >> 9;
  83 }
  84
  85 /* To allow for hash collisions, filter search continues at these
  86  * increments from the first possible entry selected by the hash. */
  87 static u16 efx_filter_increment(u32 key)
  88 {
  89         return key * 2 - 1;
  90 }
  91
  92 static enum efx_filter_table_id
  93 efx_filter_spec_table_id(const struct efx_filter_spec *spec)
  94 {
  95         BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_TCP_FULL >> 2));
  96         BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_TCP_WILD >> 2));
  97         BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_UDP_FULL >> 2));
  98         BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_UDP_WILD >> 2));
  99         BUILD_BUG_ON(EFX_FILTER_TABLE_RX_MAC != (EFX_FILTER_MAC_FULL >> 2));
 100         BUILD_BUG_ON(EFX_FILTER_TABLE_RX_MAC != (EFX_FILTER_MAC_WILD >> 2));
 101         BUILD_BUG_ON(EFX_FILTER_TABLE_TX_MAC != EFX_FILTER_TABLE_RX_MAC + 2);
 102         EFX_BUG_ON_PARANOID(spec->type == EFX_FILTER_UNSPEC);
 103         return (spec->type >> 2) + ((spec->flags & EFX_FILTER_FLAG_TX) ? 2 : 0);
 104 }
 105
 106 static struct efx_filter_table *
 107 efx_filter_spec_table(struct efx_filter_state *state,
 108                       const struct efx_filter_spec *spec)
 109 {
 110         if (spec->type == EFX_FILTER_UNSPEC)
 111                 return NULL;
 112         else
 113                 return &state->table[efx_filter_spec_table_id(spec)];
 114 }
 115
 116 static void efx_filter_table_reset_search_depth(struct efx_filter_table *table)
 117 {
 118         memset(table->search_depth, 0, sizeof(table->search_depth));
 119 }
 120
 121 static void efx_filter_push_rx_config(struct efx_nic *efx)
 122 {
 123         struct efx_filter_state *state = efx->filter_state;
 124         struct efx_filter_table *table;
 125         efx_oword_t filter_ctl;
 126
 127         efx_reado(efx, &filter_ctl, FR_BZ_RX_FILTER_CTL);
 128
 129         table = &state->table[EFX_FILTER_TABLE_RX_IP];
 130         EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_TCP_FULL_SRCH_LIMIT,
 131                             table->search_depth[EFX_FILTER_TCP_FULL] +
 132                             FILTER_CTL_SRCH_FUDGE_FULL);
 133         EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_TCP_WILD_SRCH_LIMIT,
 134                             table->search_depth[EFX_FILTER_TCP_WILD] +
 135                             FILTER_CTL_SRCH_FUDGE_WILD);
 136         EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_UDP_FULL_SRCH_LIMIT,
 137                             table->search_depth[EFX_FILTER_UDP_FULL] +
 138                             FILTER_CTL_SRCH_FUDGE_FULL);
 139         EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_UDP_WILD_SRCH_LIMIT,
 140                             table->search_depth[EFX_FILTER_UDP_WILD] +
 141                             FILTER_CTL_SRCH_FUDGE_WILD);
 142
 143         table = &state->table[EFX_FILTER_TABLE_RX_MAC];
 144         if (table->size) {
 145                 EFX_SET_OWORD_FIELD(
 146                         filter_ctl, FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT,
 147                         table->search_depth[EFX_FILTER_MAC_FULL] +
 148                         FILTER_CTL_SRCH_FUDGE_FULL);
 149                 EFX_SET_OWORD_FIELD(
 150                         filter_ctl, FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT,
 151                         table->search_depth[EFX_FILTER_MAC_WILD] +
 152                         FILTER_CTL_SRCH_FUDGE_WILD);
 153         }
 154
 155         table = &state->table[EFX_FILTER_TABLE_RX_DEF];
 156         if (table->size) {
 157                 EFX_SET_OWORD_FIELD(
 158                         filter_ctl, FRF_CZ_UNICAST_NOMATCH_Q_ID,
 159                         table->spec[EFX_FILTER_INDEX_UC_DEF].dmaq_id);
 160                 EFX_SET_OWORD_FIELD(
 161                         filter_ctl, FRF_CZ_UNICAST_NOMATCH_RSS_ENABLED,
 162                         !!(table->spec[EFX_FILTER_INDEX_UC_DEF].flags &
 163                            EFX_FILTER_FLAG_RX_RSS));
 164                 EFX_SET_OWORD_FIELD(
 165                         filter_ctl, FRF_CZ_MULTICAST_NOMATCH_Q_ID,
 166                         table->spec[EFX_FILTER_INDEX_MC_DEF].dmaq_id);
 167                 EFX_SET_OWORD_FIELD(
 168                         filter_ctl, FRF_CZ_MULTICAST_NOMATCH_RSS_ENABLED,
 169                         !!(table->spec[EFX_FILTER_INDEX_MC_DEF].flags &
 170                            EFX_FILTER_FLAG_RX_RSS));
 171         }
 172
 173         efx_writeo(efx, &filter_ctl, FR_BZ_RX_FILTER_CTL);
 174 }
 175
 176 static void efx_filter_push_tx_limits(struct efx_nic *efx)
 177 {
 178         struct efx_filter_state *state = efx->filter_state;
 179         struct efx_filter_table *table;
 180         efx_oword_t tx_cfg;
 181
 182         efx_reado(efx, &tx_cfg, FR_AZ_TX_CFG);
 183
 184         table = &state->table[EFX_FILTER_TABLE_TX_MAC];
 185         if (table->size) {
 186                 EFX_SET_OWORD_FIELD(
 187                         tx_cfg, FRF_CZ_TX_ETH_FILTER_FULL_SEARCH_RANGE,
 188                         table->search_depth[EFX_FILTER_MAC_FULL] +
 189                         FILTER_CTL_SRCH_FUDGE_FULL);
 190                 EFX_SET_OWORD_FIELD(
 191                         tx_cfg, FRF_CZ_TX_ETH_FILTER_WILD_SEARCH_RANGE,
 192                         table->search_depth[EFX_FILTER_MAC_WILD] +
 193                         FILTER_CTL_SRCH_FUDGE_WILD);
 194         }
 195
 196         efx_writeo(efx, &tx_cfg, FR_AZ_TX_CFG);
 197 }
 198
 199 static inline void __efx_filter_set_ipv4(struct efx_filter_spec *spec,
 200                                          __be32 host1, __be16 port1,
 201                                          __be32 host2, __be16 port2)
 202 {
 203         spec->data[0] = ntohl(host1) << 16 | ntohs(port1);
 204         spec->data[1] = ntohs(port2) << 16 | ntohl(host1) >> 16;
 205         spec->data[2] = ntohl(host2);
 206 }
 207
 208 static inline void __efx_filter_get_ipv4(const struct efx_filter_spec *spec,
 209                                          __be32 *host1, __be16 *port1,
 210                                          __be32 *host2, __be16 *port2)
 211 {
 212         *host1 = htonl(spec->data[0] >> 16 | spec->data[1] << 16);
 213         *port1 = htons(spec->data[0]);
 214         *host2 = htonl(spec->data[2]);
 215         *port2 = htons(spec->data[1] >> 16);
 216 }
 217
 218 /**
 219  * efx_filter_set_ipv4_local - specify IPv4 host, transport protocol and port
 220  * @spec: Specification to initialise
 221  * @proto: Transport layer protocol number
 222  * @host: Local host address (network byte order)
 223  * @port: Local port (network byte order)
 224  */
 225 int efx_filter_set_ipv4_local(struct efx_filter_spec *spec, u8 proto,
 226                               __be32 host, __be16 port)
 227 {
 228         __be32 host1;
 229         __be16 port1;
 230
 231         EFX_BUG_ON_PARANOID(!(spec->flags & EFX_FILTER_FLAG_RX));
 232
 233         /* This cannot currently be combined with other filtering */
 234         if (spec->type != EFX_FILTER_UNSPEC)
 235                 return -EPROTONOSUPPORT;
 236
 237         if (port == 0)
 238                 return -EINVAL;
 239
 240         switch (proto) {
 241         case IPPROTO_TCP:
 242                 spec->type = EFX_FILTER_TCP_WILD;
 243                 break;
 244         case IPPROTO_UDP:
 245                 spec->type = EFX_FILTER_UDP_WILD;
 246                 break;
 247         default:
 248                 return -EPROTONOSUPPORT;
 249         }
 250
 251         /* Filter is constructed in terms of source and destination,
 252          * with the odd wrinkle that the ports are swapped in a UDP
 253          * wildcard filter.  We need to convert from local and remote
 254          * (= zero for wildcard) addresses.
 255          */
 256         host1 = 0;
 257         if (proto != IPPROTO_UDP) {
 258                 port1 = 0;
 259         } else {
 260                 port1 = port;
 261                 port = 0;
 262         }
 263
 264         __efx_filter_set_ipv4(spec, host1, port1, host, port);
 265         return 0;
 266 }
 267
 268 int efx_filter_get_ipv4_local(const struct efx_filter_spec *spec,
 269                               u8 *proto, __be32 *host, __be16 *port)
 270 {
 271         __be32 host1;
 272         __be16 port1;
 273
 274         switch (spec->type) {
 275         case EFX_FILTER_TCP_WILD:
 276                 *proto = IPPROTO_TCP;
 277                 __efx_filter_get_ipv4(spec, &host1, &port1, host, port);
 278                 return 0;
 279         case EFX_FILTER_UDP_WILD:
 280                 *proto = IPPROTO_UDP;
 281                 __efx_filter_get_ipv4(spec, &host1, port, host, &port1);
 282                 return 0;
 283         default:
 284                 return -EINVAL;
 285         }
 286 }
 287
 288 /**
 289  * efx_filter_set_ipv4_full - specify IPv4 hosts, transport protocol and ports
 290  * @spec: Specification to initialise
 291  * @proto: Transport layer protocol number
 292  * @host: Local host address (network byte order)
 293  * @port: Local port (network byte order)
 294  * @rhost: Remote host address (network byte order)
 295  * @rport: Remote port (network byte order)
 296  */
 297 int efx_filter_set_ipv4_full(struct efx_filter_spec *spec, u8 proto,
 298                              __be32 host, __be16 port,
 299                              __be32 rhost, __be16 rport)
 300 {
 301         EFX_BUG_ON_PARANOID(!(spec->flags & EFX_FILTER_FLAG_RX));
 302
 303         /* This cannot currently be combined with other filtering */
 304         if (spec->type != EFX_FILTER_UNSPEC)
 305                 return -EPROTONOSUPPORT;
 306
 307         if (port == 0 || rport == 0)
 308                 return -EINVAL;
 309
 310         switch (proto) {
 311         case IPPROTO_TCP:
 312                 spec->type = EFX_FILTER_TCP_FULL;
 313                 break;
 314         case IPPROTO_UDP:
 315                 spec->type = EFX_FILTER_UDP_FULL;
 316                 break;
 317         default:
 318                 return -EPROTONOSUPPORT;
 319         }
 320
 321         __efx_filter_set_ipv4(spec, rhost, rport, host, port);
 322         return 0;
 323 }
 324
 325 int efx_filter_get_ipv4_full(const struct efx_filter_spec *spec,
 326                              u8 *proto, __be32 *host, __be16 *port,
 327                              __be32 *rhost, __be16 *rport)
 328 {
 329         switch (spec->type) {
 330         case EFX_FILTER_TCP_FULL:
 331                 *proto = IPPROTO_TCP;
 332                 break;
 333         case EFX_FILTER_UDP_FULL:
 334                 *proto = IPPROTO_UDP;
 335                 break;
 336         default:
 337                 return -EINVAL;
 338         }
 339
 340         __efx_filter_get_ipv4(spec, rhost, rport, host, port);
 341         return 0;
 342 }
 343
 344 /**
 345  * efx_filter_set_eth_local - specify local Ethernet address and optional VID
 346  * @spec: Specification to initialise
 347  * @vid: VLAN ID to match, or %EFX_FILTER_VID_UNSPEC
 348  * @addr: Local Ethernet MAC address
 349  */
 350 int efx_filter_set_eth_local(struct efx_filter_spec *spec,
 351                              u16 vid, const u8 *addr)
 352 {
 353         EFX_BUG_ON_PARANOID(!(spec->flags &
 354                               (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX)));
 355
 356         /* This cannot currently be combined with other filtering */
 357         if (spec->type != EFX_FILTER_UNSPEC)
 358                 return -EPROTONOSUPPORT;
 359
 360         if (vid == EFX_FILTER_VID_UNSPEC) {
 361                 spec->type = EFX_FILTER_MAC_WILD;
 362                 spec->data[0] = 0;
 363         } else {
 364                 spec->type = EFX_FILTER_MAC_FULL;
 365                 spec->data[0] = vid;
 366         }
 367
 368         spec->data[1] = addr[2] << 24 | addr[3] << 16 | addr[4] << 8 | addr[5];
 369         spec->data[2] = addr[0] << 8 | addr[1];
 370         return 0;
 371 }
 372
 373 /**
 374  * efx_filter_set_uc_def - specify matching otherwise-unmatched unicast
 375  * @spec: Specification to initialise
 376  */
 377 int efx_filter_set_uc_def(struct efx_filter_spec *spec)
 378 {
 379         EFX_BUG_ON_PARANOID(!(spec->flags &
 380                               (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX)));
 381
 382         if (spec->type != EFX_FILTER_UNSPEC)
 383                 return -EINVAL;
 384
 385         spec->type = EFX_FILTER_UC_DEF;
 386         memset(spec->data, 0, sizeof(spec->data)); /* ensure equality */
 387         return 0;
 388 }
 389
 390 /**
 391  * efx_filter_set_mc_def - specify matching otherwise-unmatched multicast
 392  * @spec: Specification to initialise
 393  */
 394 int efx_filter_set_mc_def(struct efx_filter_spec *spec)
 395 {
 396         EFX_BUG_ON_PARANOID(!(spec->flags &
 397                               (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX)));
 398
 399         if (spec->type != EFX_FILTER_UNSPEC)
 400                 return -EINVAL;
 401
 402         spec->type = EFX_FILTER_MC_DEF;
 403         memset(spec->data, 0, sizeof(spec->data)); /* ensure equality */
 404         return 0;
 405 }
 406
 407 static void efx_filter_reset_rx_def(struct efx_nic *efx, unsigned filter_idx)
 408 {
 409         struct efx_filter_state *state = efx->filter_state;
 410         struct efx_filter_table *table = &state->table[EFX_FILTER_TABLE_RX_DEF];
 411         struct efx_filter_spec *spec = &table->spec[filter_idx];
 412
 413         efx_filter_init_rx(spec, EFX_FILTER_PRI_MANUAL,
 414                            EFX_FILTER_FLAG_RX_RSS, 0);
 415         spec->type = EFX_FILTER_UC_DEF + filter_idx;
 416         table->used_bitmap[0] |= 1 << filter_idx;
 417 }
 418
 419 int efx_filter_get_eth_local(const struct efx_filter_spec *spec,
 420                              u16 *vid, u8 *addr)
 421 {
 422         switch (spec->type) {
 423         case EFX_FILTER_MAC_WILD:
 424                 *vid = EFX_FILTER_VID_UNSPEC;
 425                 break;
 426         case EFX_FILTER_MAC_FULL:
 427                 *vid = spec->data[0];
 428                 break;
 429         default:
 430                 return -EINVAL;
 431         }
 432
 433         addr[0] = spec->data[2] >> 8;
 434         addr[1] = spec->data[2];
 435         addr[2] = spec->data[1] >> 24;
 436         addr[3] = spec->data[1] >> 16;
 437         addr[4] = spec->data[1] >> 8;
 438         addr[5] = spec->data[1];
 439         return 0;
 440 }
 441
 442 /* Build a filter entry and return its n-tuple key. */
 443 static u32 efx_filter_build(efx_oword_t *filter, struct efx_filter_spec *spec)
 444 {
 445         u32 data3;
 446
 447         switch (efx_filter_spec_table_id(spec)) {
 448         case EFX_FILTER_TABLE_RX_IP: {
 449                 bool is_udp = (spec->type == EFX_FILTER_UDP_FULL ||
 450                                spec->type == EFX_FILTER_UDP_WILD);
 451                 EFX_POPULATE_OWORD_7(
 452                         *filter,
 453                         FRF_BZ_RSS_EN,
 454                         !!(spec->flags & EFX_FILTER_FLAG_RX_RSS),
 455                         FRF_BZ_SCATTER_EN,
 456                         !!(spec->flags & EFX_FILTER_FLAG_RX_SCATTER),
 457                         FRF_BZ_TCP_UDP, is_udp,
 458                         FRF_BZ_RXQ_ID, spec->dmaq_id,
 459                         EFX_DWORD_2, spec->data[2],
 460                         EFX_DWORD_1, spec->data[1],
 461                         EFX_DWORD_0, spec->data[0]);
 462                 data3 = is_udp;
 463                 break;
 464         }
 465
 466         case EFX_FILTER_TABLE_RX_MAC: {
 467                 bool is_wild = spec->type == EFX_FILTER_MAC_WILD;
 468                 EFX_POPULATE_OWORD_7(
 469                         *filter,
 470                         FRF_CZ_RMFT_RSS_EN,
 471                         !!(spec->flags & EFX_FILTER_FLAG_RX_RSS),
 472                         FRF_CZ_RMFT_SCATTER_EN,
 473                         !!(spec->flags & EFX_FILTER_FLAG_RX_SCATTER),
 474                         FRF_CZ_RMFT_RXQ_ID, spec->dmaq_id,
 475                         FRF_CZ_RMFT_WILDCARD_MATCH, is_wild,
 476                         FRF_CZ_RMFT_DEST_MAC_HI, spec->data[2],
 477                         FRF_CZ_RMFT_DEST_MAC_LO, spec->data[1],
 478                         FRF_CZ_RMFT_VLAN_ID, spec->data[0]);
 479                 data3 = is_wild;
 480                 break;
 481         }
 482
 483         case EFX_FILTER_TABLE_TX_MAC: {
 484                 bool is_wild = spec->type == EFX_FILTER_MAC_WILD;
 485                 EFX_POPULATE_OWORD_5(*filter,
 486                                      FRF_CZ_TMFT_TXQ_ID, spec->dmaq_id,
 487                                      FRF_CZ_TMFT_WILDCARD_MATCH, is_wild,
 488                                      FRF_CZ_TMFT_SRC_MAC_HI, spec->data[2],
 489                                      FRF_CZ_TMFT_SRC_MAC_LO, spec->data[1],
 490                                      FRF_CZ_TMFT_VLAN_ID, spec->data[0]);
 491                 data3 = is_wild | spec->dmaq_id << 1;
 492                 break;
 493         }
 494
 495         default:
 496                 BUG();
 497         }
 498
 499         return spec->data[0] ^ spec->data[1] ^ spec->data[2] ^ data3;
 500 }
 501
 502 static bool efx_filter_equal(const struct efx_filter_spec *left,
 503                              const struct efx_filter_spec *right)
 504 {
 505         if (left->type != right->type ||
 506             memcmp(left->data, right->data, sizeof(left->data)))
 507                 return false;
 508
 509         if (left->flags & EFX_FILTER_FLAG_TX &&
 510             left->dmaq_id != right->dmaq_id)
 511                 return false;
 512
 513         return true;
 514 }
 515
 516 /*
 517  * Construct/deconstruct external filter IDs.  At least the RX filter
 518  * IDs must be ordered by matching priority, for RX NFC semantics.
 519  *
 520  * Deconstruction needs to be robust against invalid IDs so that
 521  * efx_filter_remove_id_safe() and efx_filter_get_filter_safe() can
 522  * accept user-provided IDs.
 523  */
 524
 525 #define EFX_FILTER_MATCH_PRI_COUNT      5
 526
 527 static const u8 efx_filter_type_match_pri[EFX_FILTER_TYPE_COUNT] = {
 528         [EFX_FILTER_TCP_FULL]   = 0,
 529         [EFX_FILTER_UDP_FULL]   = 0,
 530         [EFX_FILTER_TCP_WILD]   = 1,
 531         [EFX_FILTER_UDP_WILD]   = 1,
 532         [EFX_FILTER_MAC_FULL]   = 2,
 533         [EFX_FILTER_MAC_WILD]   = 3,
 534         [EFX_FILTER_UC_DEF]     = 4,
 535         [EFX_FILTER_MC_DEF]     = 4,
 536 };
 537
 538 static const enum efx_filter_table_id efx_filter_range_table[] = {
 539         EFX_FILTER_TABLE_RX_IP,         /* RX match pri 0 */
 540         EFX_FILTER_TABLE_RX_IP,
 541         EFX_FILTER_TABLE_RX_MAC,
 542         EFX_FILTER_TABLE_RX_MAC,
 543         EFX_FILTER_TABLE_RX_DEF,        /* RX match pri 4 */
 544         EFX_FILTER_TABLE_COUNT,         /* TX match pri 0; invalid */
 545         EFX_FILTER_TABLE_COUNT,         /* invalid */
 546         EFX_FILTER_TABLE_TX_MAC,
 547         EFX_FILTER_TABLE_TX_MAC,        /* TX match pri 3 */
 548 };
 549
 550 #define EFX_FILTER_INDEX_WIDTH  13
 551 #define EFX_FILTER_INDEX_MASK   ((1 << EFX_FILTER_INDEX_WIDTH) - 1)
 552
 553 static inline u32
 554 efx_filter_make_id(const struct efx_filter_spec *spec, unsigned int index)
 555 {
 556         unsigned int range;
 557
 558         range = efx_filter_type_match_pri[spec->type];
 559         if (!(spec->flags & EFX_FILTER_FLAG_RX))
 560                 range += EFX_FILTER_MATCH_PRI_COUNT;
 561
 562         return range << EFX_FILTER_INDEX_WIDTH | index;
 563 }
 564
 565 static inline enum efx_filter_table_id efx_filter_id_table_id(u32 id)
 566 {
 567         unsigned int range = id >> EFX_FILTER_INDEX_WIDTH;
 568
 569         if (range < ARRAY_SIZE(efx_filter_range_table))
 570                 return efx_filter_range_table[range];
 571         else
 572                 return EFX_FILTER_TABLE_COUNT; /* invalid */
 573 }
 574
 575 static inline unsigned int efx_filter_id_index(u32 id)
 576 {
 577         return id & EFX_FILTER_INDEX_MASK;
 578 }
 579
 580 static inline u8 efx_filter_id_flags(u32 id)
 581 {
 582         unsigned int range = id >> EFX_FILTER_INDEX_WIDTH;
 583
 584         if (range < EFX_FILTER_MATCH_PRI_COUNT)
 585                 return EFX_FILTER_FLAG_RX;
 586         else
 587                 return EFX_FILTER_FLAG_TX;
 588 }
 589
 590 u32 efx_filter_get_rx_id_limit(struct efx_nic *efx)
 591 {
 592         struct efx_filter_state *state = efx->filter_state;
 593         unsigned int range = EFX_FILTER_MATCH_PRI_COUNT - 1;
 594         enum efx_filter_table_id table_id;
 595
 596         do {
 597                 table_id = efx_filter_range_table[range];
 598                 if (state->table[table_id].size != 0)
 599                         return range << EFX_FILTER_INDEX_WIDTH |
 600                                 state->table[table_id].size;
 601         } while (range--);
 602
 603         return 0;
 604 }
 605
 606 /**
 607  * efx_filter_insert_filter - add or replace a filter
 608  * @efx: NIC in which to insert the filter
 609  * @spec: Specification for the filter
 610  * @replace_equal: Flag for whether the specified filter may replace an
 611  *      existing filter with equal priority
 612  *
 613  * On success, return the filter ID.
 614  * On failure, return a negative error code.
 615  *
 616  * If an existing filter has equal match values to the new filter
 617  * spec, then the new filter might replace it, depending on the
 618  * relative priorities.  If the existing filter has lower priority, or
 619  * if @replace_equal is set and it has equal priority, then it is
 620  * replaced.  Otherwise the function fails, returning -%EPERM if
 621  * the existing filter has higher priority or -%EEXIST if it has
 622  * equal priority.
 623  */
 624 s32 efx_filter_insert_filter(struct efx_nic *efx, struct efx_filter_spec *spec,
 625                              bool replace_equal)
 626 {
 627         struct efx_filter_state *state = efx->filter_state;
 628         struct efx_filter_table *table = efx_filter_spec_table(state, spec);
 629         struct efx_filter_spec *saved_spec;
 630         efx_oword_t filter;
 631         unsigned int filter_idx, depth = 0;
 632         int rc;
 633
 634         if (!table || table->size == 0)
 635                 return -EINVAL;
 636
 637         netif_vdbg(efx, hw, efx->net_dev,
 638                    "%s: type %d search_depth=%d", __func__, spec->type,
 639                    table->search_depth[spec->type]);
 640
 641         if (table->id == EFX_FILTER_TABLE_RX_DEF) {
 642                 /* One filter spec per type */
 643                 BUILD_BUG_ON(EFX_FILTER_INDEX_UC_DEF != 0);
 644                 BUILD_BUG_ON(EFX_FILTER_INDEX_MC_DEF !=
 645                              EFX_FILTER_MC_DEF - EFX_FILTER_UC_DEF);
 646                 filter_idx = spec->type - EFX_FILTER_INDEX_UC_DEF;
 647
 648                 spin_lock_bh(&state->lock);
 649         } else {
 650                 u32 key = efx_filter_build(&filter, spec);
 651                 unsigned int hash = efx_filter_hash(key);
 652                 unsigned int incr = efx_filter_increment(key);
 653                 unsigned int depth_max =
 654                         spec->priority <= EFX_FILTER_PRI_HINT ?
 655                         FILTER_CTL_SRCH_HINT_MAX : FILTER_CTL_SRCH_MAX;
 656
 657                 filter_idx = hash & (table->size - 1);
 658                 depth = 1;
 659
 660                 spin_lock_bh(&state->lock);
 661
 662                 for (;;) {
 663                         if (!test_bit(filter_idx, table->used_bitmap) ||
 664                             efx_filter_equal(spec, &table->spec[filter_idx]))
 665                                 break;
 666
 667                         /* Return failure if we reached the maximum search
 668                          * depth
 669                          */
 670                         if (depth == depth_max) {
 671                                 rc = -EBUSY;
 672                                 goto out;
 673                         }
 674
 675                         filter_idx = (filter_idx + incr) & (table->size - 1);
 676                         ++depth;
 677                 }
 678         }
 679
 680         saved_spec = &table->spec[filter_idx];
 681
 682         if (test_bit(filter_idx, table->used_bitmap)) {
 683                 /* Should we replace the existing filter? */
 684                 if (spec->priority == saved_spec->priority && !replace_equal) {
 685                         rc = -EEXIST;
 686                         goto out;
 687                 }
 688                 if (spec->priority < saved_spec->priority) {
 689                         rc = -EPERM;
 690                         goto out;
 691                 }
 692         } else {
 693                 __set_bit(filter_idx, table->used_bitmap);
 694                 ++table->used;
 695         }
 696         *saved_spec = *spec;
 697
 698         if (table->id == EFX_FILTER_TABLE_RX_DEF) {
 699                 efx_filter_push_rx_config(efx);
 700         } else {
 701                 if (table->search_depth[spec->type] < depth) {
 702                         table->search_depth[spec->type] = depth;
 703                         if (spec->flags & EFX_FILTER_FLAG_TX)
 704                                 efx_filter_push_tx_limits(efx);
 705                         else
 706                                 efx_filter_push_rx_config(efx);
 707                 }
 708
 709                 efx_writeo(efx, &filter,
 710                            table->offset + table->step * filter_idx);
 711         }
 712
 713         netif_vdbg(efx, hw, efx->net_dev,
 714                    "%s: filter type %d index %d rxq %u set",
 715                    __func__, spec->type, filter_idx, spec->dmaq_id);
 716         rc = efx_filter_make_id(spec, filter_idx);
 717
 718 out:
 719         spin_unlock_bh(&state->lock);
 720         return rc;
 721 }
 722
 723 static void efx_filter_table_clear_entry(struct efx_nic *efx,
 724                                          struct efx_filter_table *table,
 725                                          unsigned int filter_idx)
 726 {
 727         static efx_oword_t filter;
 728
 729         if (table->id == EFX_FILTER_TABLE_RX_DEF) {
 730                 /* RX default filters must always exist */
 731                 efx_filter_reset_rx_def(efx, filter_idx);
 732                 efx_filter_push_rx_config(efx);
 733         } else if (test_bit(filter_idx, table->used_bitmap)) {
 734                 __clear_bit(filter_idx, table->used_bitmap);
 735                 --table->used;
 736                 memset(&table->spec[filter_idx], 0, sizeof(table->spec[0]));
 737
 738                 efx_writeo(efx, &filter,
 739                            table->offset + table->step * filter_idx);
 740         }
 741 }
 742
 743 /**
 744  * efx_filter_remove_id_safe - remove a filter by ID, carefully
 745  * @efx: NIC from which to remove the filter
 746  * @priority: Priority of filter, as passed to @efx_filter_insert_filter
 747  * @filter_id: ID of filter, as returned by @efx_filter_insert_filter
 748  *
 749  * This function will range-check @filter_id, so it is safe to call
 750  * with a value passed from userland.
 751  */
 752 int efx_filter_remove_id_safe(struct efx_nic *efx,
 753                               enum efx_filter_priority priority,
 754                               u32 filter_id)
 755 {
 756         struct efx_filter_state *state = efx->filter_state;
 757         enum efx_filter_table_id table_id;
 758         struct efx_filter_table *table;
 759         unsigned int filter_idx;
 760         struct efx_filter_spec *spec;
 761         u8 filter_flags;
 762         int rc;
 763
 764         table_id = efx_filter_id_table_id(filter_id);
 765         if ((unsigned int)table_id >= EFX_FILTER_TABLE_COUNT)
 766                 return -ENOENT;
 767         table = &state->table[table_id];
 768
 769         filter_idx = efx_filter_id_index(filter_id);
 770         if (filter_idx >= table->size)
 771                 return -ENOENT;
 772         spec = &table->spec[filter_idx];
 773
 774         filter_flags = efx_filter_id_flags(filter_id);
 775
 776         spin_lock_bh(&state->lock);
 777
 778         if (test_bit(filter_idx, table->used_bitmap) &&
 779             spec->priority == priority) {
 780                 efx_filter_table_clear_entry(efx, table, filter_idx);
 781                 if (table->used == 0)
 782                         efx_filter_table_reset_search_depth(table);
 783                 rc = 0;
 784         } else {
 785                 rc = -ENOENT;
 786         }
 787
 788         spin_unlock_bh(&state->lock);
 789
 790         return rc;
 791 }
 792
 793 /**
 794  * efx_filter_get_filter_safe - retrieve a filter by ID, carefully
 795  * @efx: NIC from which to remove the filter
 796  * @priority: Priority of filter, as passed to @efx_filter_insert_filter
 797  * @filter_id: ID of filter, as returned by @efx_filter_insert_filter
 798  * @spec: Buffer in which to store filter specification
 799  *
 800  * This function will range-check @filter_id, so it is safe to call
 801  * with a value passed from userland.
 802  */
 803 int efx_filter_get_filter_safe(struct efx_nic *efx,
 804                                enum efx_filter_priority priority,
 805                                u32 filter_id, struct efx_filter_spec *spec_buf)
 806 {
 807         struct efx_filter_state *state = efx->filter_state;
 808         enum efx_filter_table_id table_id;
 809         struct efx_filter_table *table;
 810         struct efx_filter_spec *spec;
 811         unsigned int filter_idx;
 812         u8 filter_flags;
 813         int rc;
 814
 815         table_id = efx_filter_id_table_id(filter_id);
 816         if ((unsigned int)table_id >= EFX_FILTER_TABLE_COUNT)
 817                 return -ENOENT;
 818         table = &state->table[table_id];
 819
 820         filter_idx = efx_filter_id_index(filter_id);
 821         if (filter_idx >= table->size)
 822                 return -ENOENT;
 823         spec = &table->spec[filter_idx];
 824
 825         filter_flags = efx_filter_id_flags(filter_id);
 826
 827         spin_lock_bh(&state->lock);
 828
 829         if (test_bit(filter_idx, table->used_bitmap) &&
 830             spec->priority == priority) {
 831                 *spec_buf = *spec;
 832                 rc = 0;
 833         } else {
 834                 rc = -ENOENT;
 835         }
 836
 837         spin_unlock_bh(&state->lock);
 838
 839         return rc;
 840 }
 841
 842 static void efx_filter_table_clear(struct efx_nic *efx,
 843                                    enum efx_filter_table_id table_id,
 844                                    enum efx_filter_priority priority)
 845 {
 846         struct efx_filter_state *state = efx->filter_state;
 847         struct efx_filter_table *table = &state->table[table_id];
 848         unsigned int filter_idx;
 849
 850         spin_lock_bh(&state->lock);
 851
 852         for (filter_idx = 0; filter_idx < table->size; ++filter_idx)
 853                 if (table->spec[filter_idx].priority <= priority)
 854                         efx_filter_table_clear_entry(efx, table, filter_idx);
 855         if (table->used == 0)
 856                 efx_filter_table_reset_search_depth(table);
 857
 858         spin_unlock_bh(&state->lock);
 859 }
 860
 861 /**
 862  * efx_filter_clear_rx - remove RX filters by priority
 863  * @efx: NIC from which to remove the filters
 864  * @priority: Maximum priority to remove
 865  */
 866 void efx_filter_clear_rx(struct efx_nic *efx, enum efx_filter_priority priority)
 867 {
 868         efx_filter_table_clear(efx, EFX_FILTER_TABLE_RX_IP, priority);
 869         efx_filter_table_clear(efx, EFX_FILTER_TABLE_RX_MAC, priority);
 870 }
 871
 872 u32 efx_filter_count_rx_used(struct efx_nic *efx,
 873                              enum efx_filter_priority priority)
 874 {
 875         struct efx_filter_state *state = efx->filter_state;
 876         enum efx_filter_table_id table_id;
 877         struct efx_filter_table *table;
 878         unsigned int filter_idx;
 879         u32 count = 0;
 880
 881         spin_lock_bh(&state->lock);
 882
 883         for (table_id = EFX_FILTER_TABLE_RX_IP;
 884              table_id <= EFX_FILTER_TABLE_RX_DEF;
 885              table_id++) {
 886                 table = &state->table[table_id];
 887                 for (filter_idx = 0; filter_idx < table->size; filter_idx++) {
 888                         if (test_bit(filter_idx, table->used_bitmap) &&
 889                             table->spec[filter_idx].priority == priority)
 890                                 ++count;
 891                 }
 892         }
 893
 894         spin_unlock_bh(&state->lock);
 895
 896         return count;
 897 }
 898
 899 s32 efx_filter_get_rx_ids(struct efx_nic *efx,
 900                           enum efx_filter_priority priority,
 901                           u32 *buf, u32 size)
 902 {
 903         struct efx_filter_state *state = efx->filter_state;
 904         enum efx_filter_table_id table_id;
 905         struct efx_filter_table *table;
 906         unsigned int filter_idx;
 907         s32 count = 0;
 908
 909         spin_lock_bh(&state->lock);
 910
 911         for (table_id = EFX_FILTER_TABLE_RX_IP;
 912              table_id <= EFX_FILTER_TABLE_RX_DEF;
 913              table_id++) {
 914                 table = &state->table[table_id];
 915                 for (filter_idx = 0; filter_idx < table->size; filter_idx++) {
 916                         if (test_bit(filter_idx, table->used_bitmap) &&
 917                             table->spec[filter_idx].priority == priority) {
 918                                 if (count == size) {
 919                                         count = -EMSGSIZE;
 920                                         goto out;
 921                                 }
 922                                 buf[count++] = efx_filter_make_id(
 923                                         &table->spec[filter_idx], filter_idx);
 924                         }
 925                 }
 926         }
 927 out:
 928         spin_unlock_bh(&state->lock);
 929
 930         return count;
 931 }
 932
 933 /* Restore filter stater after reset */
 934 void efx_restore_filters(struct efx_nic *efx)
 935 {
 936         struct efx_filter_state *state = efx->filter_state;
 937         enum efx_filter_table_id table_id;
 938         struct efx_filter_table *table;
 939         efx_oword_t filter;
 940         unsigned int filter_idx;
 941
 942         spin_lock_bh(&state->lock);
 943
 944         for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) {
 945                 table = &state->table[table_id];
 946
 947                 /* Check whether this is a regular register table */
 948                 if (table->step == 0)
 949                         continue;
 950
 951                 for (filter_idx = 0; filter_idx < table->size; filter_idx++) {
 952                         if (!test_bit(filter_idx, table->used_bitmap))
 953                                 continue;
 954                         efx_filter_build(&filter, &table->spec[filter_idx]);
 955                         efx_writeo(efx, &filter,
 956                                    table->offset + table->step * filter_idx);
 957                 }
 958         }
 959
 960         efx_filter_push_rx_config(efx);
 961         efx_filter_push_tx_limits(efx);
 962
 963         spin_unlock_bh(&state->lock);
 964 }
 965
 966 int efx_probe_filters(struct efx_nic *efx)
 967 {
 968         struct efx_filter_state *state;
 969         struct efx_filter_table *table;
 970         unsigned table_id;
 971
 972         state = kzalloc(sizeof(*efx->filter_state), GFP_KERNEL);
 973         if (!state)
 974                 return -ENOMEM;
 975         efx->filter_state = state;
 976
 977         spin_lock_init(&state->lock);
 978
 979         if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) {
 980 #ifdef CONFIG_RFS_ACCEL
 981                 state->rps_flow_id = kcalloc(FR_BZ_RX_FILTER_TBL0_ROWS,
 982                                              sizeof(*state->rps_flow_id),
 983                                              GFP_KERNEL);
 984                 if (!state->rps_flow_id)
 985                         goto fail;
 986 #endif
 987                 table = &state->table[EFX_FILTER_TABLE_RX_IP];
 988                 table->id = EFX_FILTER_TABLE_RX_IP;
 989                 table->offset = FR_BZ_RX_FILTER_TBL0;
 990                 table->size = FR_BZ_RX_FILTER_TBL0_ROWS;
 991                 table->step = FR_BZ_RX_FILTER_TBL0_STEP;
 992         }
 993
 994         if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0) {
 995                 table = &state->table[EFX_FILTER_TABLE_RX_MAC];
 996                 table->id = EFX_FILTER_TABLE_RX_MAC;
 997                 table->offset = FR_CZ_RX_MAC_FILTER_TBL0;
 998                 table->size = FR_CZ_RX_MAC_FILTER_TBL0_ROWS;
 999                 table->step = FR_CZ_RX_MAC_FILTER_TBL0_STEP;
1000
1001                 table = &state->table[EFX_FILTER_TABLE_RX_DEF];
1002                 table->id = EFX_FILTER_TABLE_RX_DEF;
1003                 table->size = EFX_FILTER_SIZE_RX_DEF;
1004
1005                 table = &state->table[EFX_FILTER_TABLE_TX_MAC];
1006                 table->id = EFX_FILTER_TABLE_TX_MAC;
1007                 table->offset = FR_CZ_TX_MAC_FILTER_TBL0;
1008                 table->size = FR_CZ_TX_MAC_FILTER_TBL0_ROWS;
1009                 table->step = FR_CZ_TX_MAC_FILTER_TBL0_STEP;
1010         }
1011
1012         for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) {
1013                 table = &state->table[table_id];
1014                 if (table->size == 0)
1015                         continue;
1016                 table->used_bitmap = kcalloc(BITS_TO_LONGS(table->size),
1017                                              sizeof(unsigned long),
1018                                              GFP_KERNEL);
1019                 if (!table->used_bitmap)
1020                         goto fail;
1021                 table->spec = vzalloc(table->size * sizeof(*table->spec));
1022                 if (!table->spec)
1023                         goto fail;
1024         }
1025
1026         if (state->table[EFX_FILTER_TABLE_RX_DEF].size) {
1027                 /* RX default filters must always exist */
1028                 unsigned i;
1029                 for (i = 0; i < EFX_FILTER_SIZE_RX_DEF; i++)
1030                         efx_filter_reset_rx_def(efx, i);
1031         }
1032
1033         efx_filter_push_rx_config(efx);
1034
1035         return 0;
1036
1037 fail:
1038         efx_remove_filters(efx);
1039         return -ENOMEM;
1040 }
1041
1042 void efx_remove_filters(struct efx_nic *efx)
1043 {
1044         struct efx_filter_state *state = efx->filter_state;
1045         enum efx_filter_table_id table_id;
1046
1047         for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) {
1048                 kfree(state->table[table_id].used_bitmap);
1049                 vfree(state->table[table_id].spec);
1050         }
1051 #ifdef CONFIG_RFS_ACCEL
1052         kfree(state->rps_flow_id);
1053 #endif
1054         kfree(state);
1055 }
1056
1057 #ifdef CONFIG_RFS_ACCEL
1058
1059 int efx_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb,
1060                    u16 rxq_index, u32 flow_id)
1061 {
1062         struct efx_nic *efx = netdev_priv(net_dev);
1063         struct efx_channel *channel;
1064         struct efx_filter_state *state = efx->filter_state;
1065         struct efx_filter_spec spec;
1066         const struct iphdr *ip;
1067         const __be16 *ports;
1068         int nhoff;
1069         int rc;
1070
1071         nhoff = skb_network_offset(skb);
1072
1073         if (skb->protocol != htons(ETH_P_IP))
1074                 return -EPROTONOSUPPORT;
1075
1076         /* RFS must validate the IP header length before calling us */
1077         EFX_BUG_ON_PARANOID(skb_headlen(skb) < nhoff + sizeof(*ip));
1078         ip = (const struct iphdr *)(skb->data + nhoff);
1079         if (ip_is_fragment(ip))
1080                 return -EPROTONOSUPPORT;
1081         EFX_BUG_ON_PARANOID(skb_headlen(skb) < nhoff + 4 * ip->ihl + 4);
1082         ports = (const __be16 *)(skb->data + nhoff + 4 * ip->ihl);
1083
1084         efx_filter_init_rx(&spec, EFX_FILTER_PRI_HINT, 0, rxq_index);
1085         rc = efx_filter_set_ipv4_full(&spec, ip->protocol,
1086                                       ip->daddr, ports[1], ip->saddr, ports[0]);
1087         if (rc)
1088                 return rc;
1089
1090         rc = efx_filter_insert_filter(efx, &spec, true);
1091         if (rc < 0)
1092                 return rc;
1093
1094         /* Remember this so we can check whether to expire the filter later */
1095         state->rps_flow_id[rc] = flow_id;
1096         channel = efx_get_channel(efx, skb_get_rx_queue(skb));
1097         ++channel->rfs_filters_added;
1098
1099         netif_info(efx, rx_status, efx->net_dev,
1100                    "steering %s %pI4:%u:%pI4:%u to queue %u [flow %u filter %d]\n",
1101                    (ip->protocol == IPPROTO_TCP) ? "TCP" : "UDP",
1102                    &ip->saddr, ntohs(ports[0]), &ip->daddr, ntohs(ports[1]),
1103                    rxq_index, flow_id, rc);
1104
1105         return rc;
1106 }
1107
1108 bool __efx_filter_rfs_expire(struct efx_nic *efx, unsigned quota)
1109 {
1110         struct efx_filter_state *state = efx->filter_state;
1111         struct efx_filter_table *table = &state->table[EFX_FILTER_TABLE_RX_IP];
1112         unsigned mask = table->size - 1;
1113         unsigned index;
1114         unsigned stop;
1115
1116         if (!spin_trylock_bh(&state->lock))
1117                 return false;
1118
1119         index = state->rps_expire_index;
1120         stop = (index + quota) & mask;
1121
1122         while (index != stop) {
1123                 if (test_bit(index, table->used_bitmap) &&
1124                     table->spec[index].priority == EFX_FILTER_PRI_HINT &&
1125                     rps_may_expire_flow(efx->net_dev,
1126                                         table->spec[index].dmaq_id,
1127                                         state->rps_flow_id[index], index)) {
1128                         netif_info(efx, rx_status, efx->net_dev,
1129                                    "expiring filter %d [flow %u]\n",
1130                                    index, state->rps_flow_id[index]);
1131                         efx_filter_table_clear_entry(efx, table, index);
1132                 }
1133                 index = (index + 1) & mask;
1134         }
1135
1136         state->rps_expire_index = stop;
1137         if (table->used == 0)
1138                 efx_filter_table_reset_search_depth(table);
1139
1140         spin_unlock_bh(&state->lock);
1141         return true;
1142 }
1143
1144 #endif /* CONFIG_RFS_ACCEL */