drivers/net/ethernet/sfc/filter.c

   1 /****************************************************************************
   2  * Driver for Solarflare Solarstorm network controllers and boards
   3  * Copyright 2005-2010 Solarflare Communications Inc.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 as published
   7  * by the Free Software Foundation, incorporated herein by reference.
   8  */
   9
  10 #include <linux/in.h>
  11 #include <net/ip.h>
  12 #include "efx.h"
  13 #include "filter.h"
  14 #include "io.h"
  15 #include "nic.h"
  16 #include "regs.h"
  17
  18 /* "Fudge factors" - difference between programmed value and actual depth.
  19  * Due to pipelined implementation we need to program H/W with a value that
  20  * is larger than the hop limit we want.
  21  */
  22 #define FILTER_CTL_SRCH_FUDGE_WILD 3
  23 #define FILTER_CTL_SRCH_FUDGE_FULL 1
  24
  25 /* Hard maximum hop limit.  Hardware will time-out beyond 200-something.
  26  * We also need to avoid infinite loops in efx_filter_search() when the
  27  * table is full.
  28  */
  29 #define FILTER_CTL_SRCH_MAX 200
  30
  31 /* Don't try very hard to find space for performance hints, as this is
  32  * counter-productive. */
  33 #define FILTER_CTL_SRCH_HINT_MAX 5
  34
  35 enum efx_filter_table_id {
  36         EFX_FILTER_TABLE_RX_IP = 0,
  37         EFX_FILTER_TABLE_RX_MAC,
  38         EFX_FILTER_TABLE_RX_DEF,
  39         EFX_FILTER_TABLE_TX_MAC,
  40         EFX_FILTER_TABLE_COUNT,
  41 };
  42
  43 enum efx_filter_index {
  44         EFX_FILTER_INDEX_UC_DEF,
  45         EFX_FILTER_INDEX_MC_DEF,
  46         EFX_FILTER_SIZE_RX_DEF,
  47 };
  48
  49 struct efx_filter_table {
  50         enum efx_filter_table_id id;
  51         u32             offset;         /* address of table relative to BAR */
  52         unsigned        size;           /* number of entries */
  53         unsigned        step;           /* step between entries */
  54         unsigned        used;           /* number currently used */
  55         unsigned long   *used_bitmap;
  56         struct efx_filter_spec *spec;
  57         unsigned        search_depth[EFX_FILTER_TYPE_COUNT];
  58 };
  59
  60 struct efx_filter_state {
  61         spinlock_t      lock;
  62         struct efx_filter_table table[EFX_FILTER_TABLE_COUNT];
  63 #ifdef CONFIG_RFS_ACCEL
  64         u32             *rps_flow_id;
  65         unsigned        rps_expire_index;
  66 #endif
  67 };
  68
  69 static void efx_filter_table_clear_entry(struct efx_nic *efx,
  70                                          struct efx_filter_table *table,
  71                                          unsigned int filter_idx);
  72
  73 /* The filter hash function is LFSR polynomial x^16 + x^3 + 1 of a 32-bit
  74  * key derived from the n-tuple.  The initial LFSR state is 0xffff. */
  75 static u16 efx_filter_hash(u32 key)
  76 {
  77         u16 tmp;
  78
  79         /* First 16 rounds */
  80         tmp = 0x1fff ^ key >> 16;
  81         tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
  82         tmp = tmp ^ tmp >> 9;
  83         /* Last 16 rounds */
  84         tmp = tmp ^ tmp << 13 ^ key;
  85         tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
  86         return tmp ^ tmp >> 9;
  87 }
  88
  89 /* To allow for hash collisions, filter search continues at these
  90  * increments from the first possible entry selected by the hash. */
  91 static u16 efx_filter_increment(u32 key)
  92 {
  93         return key * 2 - 1;
  94 }
  95
  96 static enum efx_filter_table_id
  97 efx_filter_spec_table_id(const struct efx_filter_spec *spec)
  98 {
  99         BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_TCP_FULL >> 2));
 100         BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_TCP_WILD >> 2));
 101         BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_UDP_FULL >> 2));
 102         BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_UDP_WILD >> 2));
 103         BUILD_BUG_ON(EFX_FILTER_TABLE_RX_MAC != (EFX_FILTER_MAC_FULL >> 2));
 104         BUILD_BUG_ON(EFX_FILTER_TABLE_RX_MAC != (EFX_FILTER_MAC_WILD >> 2));
 105         BUILD_BUG_ON(EFX_FILTER_TABLE_TX_MAC != EFX_FILTER_TABLE_RX_MAC + 2);
 106         EFX_BUG_ON_PARANOID(spec->type == EFX_FILTER_UNSPEC);
 107         return (spec->type >> 2) + ((spec->flags & EFX_FILTER_FLAG_TX) ? 2 : 0);
 108 }
 109
 110 static struct efx_filter_table *
 111 efx_filter_spec_table(struct efx_filter_state *state,
 112                       const struct efx_filter_spec *spec)
 113 {
 114         if (spec->type == EFX_FILTER_UNSPEC)
 115                 return NULL;
 116         else
 117                 return &state->table[efx_filter_spec_table_id(spec)];
 118 }
 119
 120 static void efx_filter_table_reset_search_depth(struct efx_filter_table *table)
 121 {
 122         memset(table->search_depth, 0, sizeof(table->search_depth));
 123 }
 124
 125 static void efx_filter_push_rx_config(struct efx_nic *efx)
 126 {
 127         struct efx_filter_state *state = efx->filter_state;
 128         struct efx_filter_table *table;
 129         efx_oword_t filter_ctl;
 130
 131         efx_reado(efx, &filter_ctl, FR_BZ_RX_FILTER_CTL);
 132
 133         table = &state->table[EFX_FILTER_TABLE_RX_IP];
 134         EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_TCP_FULL_SRCH_LIMIT,
 135                             table->search_depth[EFX_FILTER_TCP_FULL] +
 136                             FILTER_CTL_SRCH_FUDGE_FULL);
 137         EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_TCP_WILD_SRCH_LIMIT,
 138                             table->search_depth[EFX_FILTER_TCP_WILD] +
 139                             FILTER_CTL_SRCH_FUDGE_WILD);
 140         EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_UDP_FULL_SRCH_LIMIT,
 141                             table->search_depth[EFX_FILTER_UDP_FULL] +
 142                             FILTER_CTL_SRCH_FUDGE_FULL);
 143         EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_UDP_WILD_SRCH_LIMIT,
 144                             table->search_depth[EFX_FILTER_UDP_WILD] +
 145                             FILTER_CTL_SRCH_FUDGE_WILD);
 146
 147         table = &state->table[EFX_FILTER_TABLE_RX_MAC];
 148         if (table->size) {
 149                 EFX_SET_OWORD_FIELD(
 150                         filter_ctl, FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT,
 151                         table->search_depth[EFX_FILTER_MAC_FULL] +
 152                         FILTER_CTL_SRCH_FUDGE_FULL);
 153                 EFX_SET_OWORD_FIELD(
 154                         filter_ctl, FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT,
 155                         table->search_depth[EFX_FILTER_MAC_WILD] +
 156                         FILTER_CTL_SRCH_FUDGE_WILD);
 157         }
 158
 159         table = &state->table[EFX_FILTER_TABLE_RX_DEF];
 160         if (table->size) {
 161                 EFX_SET_OWORD_FIELD(
 162                         filter_ctl, FRF_CZ_UNICAST_NOMATCH_Q_ID,
 163                         table->spec[EFX_FILTER_INDEX_UC_DEF].dmaq_id);
 164                 EFX_SET_OWORD_FIELD(
 165                         filter_ctl, FRF_CZ_UNICAST_NOMATCH_RSS_ENABLED,
 166                         !!(table->spec[EFX_FILTER_INDEX_UC_DEF].flags &
 167                            EFX_FILTER_FLAG_RX_RSS));
 168                 EFX_SET_OWORD_FIELD(
 169                         filter_ctl, FRF_CZ_MULTICAST_NOMATCH_Q_ID,
 170                         table->spec[EFX_FILTER_INDEX_MC_DEF].dmaq_id);
 171                 EFX_SET_OWORD_FIELD(
 172                         filter_ctl, FRF_CZ_MULTICAST_NOMATCH_RSS_ENABLED,
 173                         !!(table->spec[EFX_FILTER_INDEX_MC_DEF].flags &
 174                            EFX_FILTER_FLAG_RX_RSS));
 175         }
 176
 177         efx_writeo(efx, &filter_ctl, FR_BZ_RX_FILTER_CTL);
 178 }
 179
 180 static void efx_filter_push_tx_limits(struct efx_nic *efx)
 181 {
 182         struct efx_filter_state *state = efx->filter_state;
 183         struct efx_filter_table *table;
 184         efx_oword_t tx_cfg;
 185
 186         efx_reado(efx, &tx_cfg, FR_AZ_TX_CFG);
 187
 188         table = &state->table[EFX_FILTER_TABLE_TX_MAC];
 189         if (table->size) {
 190                 EFX_SET_OWORD_FIELD(
 191                         tx_cfg, FRF_CZ_TX_ETH_FILTER_FULL_SEARCH_RANGE,
 192                         table->search_depth[EFX_FILTER_MAC_FULL] +
 193                         FILTER_CTL_SRCH_FUDGE_FULL);
 194                 EFX_SET_OWORD_FIELD(
 195                         tx_cfg, FRF_CZ_TX_ETH_FILTER_WILD_SEARCH_RANGE,
 196                         table->search_depth[EFX_FILTER_MAC_WILD] +
 197                         FILTER_CTL_SRCH_FUDGE_WILD);
 198         }
 199
 200         efx_writeo(efx, &tx_cfg, FR_AZ_TX_CFG);
 201 }
 202
 203 static inline void __efx_filter_set_ipv4(struct efx_filter_spec *spec,
 204                                          __be32 host1, __be16 port1,
 205                                          __be32 host2, __be16 port2)
 206 {
 207         spec->data[0] = ntohl(host1) << 16 | ntohs(port1);
 208         spec->data[1] = ntohs(port2) << 16 | ntohl(host1) >> 16;
 209         spec->data[2] = ntohl(host2);
 210 }
 211
 212 static inline void __efx_filter_get_ipv4(const struct efx_filter_spec *spec,
 213                                          __be32 *host1, __be16 *port1,
 214                                          __be32 *host2, __be16 *port2)
 215 {
 216         *host1 = htonl(spec->data[0] >> 16 | spec->data[1] << 16);
 217         *port1 = htons(spec->data[0]);
 218         *host2 = htonl(spec->data[2]);
 219         *port2 = htons(spec->data[1] >> 16);
 220 }
 221
 222 /**
 223  * efx_filter_set_ipv4_local - specify IPv4 host, transport protocol and port
 224  * @spec: Specification to initialise
 225  * @proto: Transport layer protocol number
 226  * @host: Local host address (network byte order)
 227  * @port: Local port (network byte order)
 228  */
 229 int efx_filter_set_ipv4_local(struct efx_filter_spec *spec, u8 proto,
 230                               __be32 host, __be16 port)
 231 {
 232         __be32 host1;
 233         __be16 port1;
 234
 235         EFX_BUG_ON_PARANOID(!(spec->flags & EFX_FILTER_FLAG_RX));
 236
 237         /* This cannot currently be combined with other filtering */
 238         if (spec->type != EFX_FILTER_UNSPEC)
 239                 return -EPROTONOSUPPORT;
 240
 241         if (port == 0)
 242                 return -EINVAL;
 243
 244         switch (proto) {
 245         case IPPROTO_TCP:
 246                 spec->type = EFX_FILTER_TCP_WILD;
 247                 break;
 248         case IPPROTO_UDP:
 249                 spec->type = EFX_FILTER_UDP_WILD;
 250                 break;
 251         default:
 252                 return -EPROTONOSUPPORT;
 253         }
 254
 255         /* Filter is constructed in terms of source and destination,
 256          * with the odd wrinkle that the ports are swapped in a UDP
 257          * wildcard filter.  We need to convert from local and remote
 258          * (= zero for wildcard) addresses.
 259          */
 260         host1 = 0;
 261         if (proto != IPPROTO_UDP) {
 262                 port1 = 0;
 263         } else {
 264                 port1 = port;
 265                 port = 0;
 266         }
 267
 268         __efx_filter_set_ipv4(spec, host1, port1, host, port);
 269         return 0;
 270 }
 271
 272 int efx_filter_get_ipv4_local(const struct efx_filter_spec *spec,
 273                               u8 *proto, __be32 *host, __be16 *port)
 274 {
 275         __be32 host1;
 276         __be16 port1;
 277
 278         switch (spec->type) {
 279         case EFX_FILTER_TCP_WILD:
 280                 *proto = IPPROTO_TCP;
 281                 __efx_filter_get_ipv4(spec, &host1, &port1, host, port);
 282                 return 0;
 283         case EFX_FILTER_UDP_WILD:
 284                 *proto = IPPROTO_UDP;
 285                 __efx_filter_get_ipv4(spec, &host1, port, host, &port1);
 286                 return 0;
 287         default:
 288                 return -EINVAL;
 289         }
 290 }
 291
 292 /**
 293  * efx_filter_set_ipv4_full - specify IPv4 hosts, transport protocol and ports
 294  * @spec: Specification to initialise
 295  * @proto: Transport layer protocol number
 296  * @host: Local host address (network byte order)
 297  * @port: Local port (network byte order)
 298  * @rhost: Remote host address (network byte order)
 299  * @rport: Remote port (network byte order)
 300  */
 301 int efx_filter_set_ipv4_full(struct efx_filter_spec *spec, u8 proto,
 302                              __be32 host, __be16 port,
 303                              __be32 rhost, __be16 rport)
 304 {
 305         EFX_BUG_ON_PARANOID(!(spec->flags & EFX_FILTER_FLAG_RX));
 306
 307         /* This cannot currently be combined with other filtering */
 308         if (spec->type != EFX_FILTER_UNSPEC)
 309                 return -EPROTONOSUPPORT;
 310
 311         if (port == 0 || rport == 0)
 312                 return -EINVAL;
 313
 314         switch (proto) {
 315         case IPPROTO_TCP:
 316                 spec->type = EFX_FILTER_TCP_FULL;
 317                 break;
 318         case IPPROTO_UDP:
 319                 spec->type = EFX_FILTER_UDP_FULL;
 320                 break;
 321         default:
 322                 return -EPROTONOSUPPORT;
 323         }
 324
 325         __efx_filter_set_ipv4(spec, rhost, rport, host, port);
 326         return 0;
 327 }
 328
 329 int efx_filter_get_ipv4_full(const struct efx_filter_spec *spec,
 330                              u8 *proto, __be32 *host, __be16 *port,
 331                              __be32 *rhost, __be16 *rport)
 332 {
 333         switch (spec->type) {
 334         case EFX_FILTER_TCP_FULL:
 335                 *proto = IPPROTO_TCP;
 336                 break;
 337         case EFX_FILTER_UDP_FULL:
 338                 *proto = IPPROTO_UDP;
 339                 break;
 340         default:
 341                 return -EINVAL;
 342         }
 343
 344         __efx_filter_get_ipv4(spec, rhost, rport, host, port);
 345         return 0;
 346 }
 347
 348 /**
 349  * efx_filter_set_eth_local - specify local Ethernet address and optional VID
 350  * @spec: Specification to initialise
 351  * @vid: VLAN ID to match, or %EFX_FILTER_VID_UNSPEC
 352  * @addr: Local Ethernet MAC address
 353  */
 354 int efx_filter_set_eth_local(struct efx_filter_spec *spec,
 355                              u16 vid, const u8 *addr)
 356 {
 357         EFX_BUG_ON_PARANOID(!(spec->flags &
 358                               (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX)));
 359
 360         /* This cannot currently be combined with other filtering */
 361         if (spec->type != EFX_FILTER_UNSPEC)
 362                 return -EPROTONOSUPPORT;
 363
 364         if (vid == EFX_FILTER_VID_UNSPEC) {
 365                 spec->type = EFX_FILTER_MAC_WILD;
 366                 spec->data[0] = 0;
 367         } else {
 368                 spec->type = EFX_FILTER_MAC_FULL;
 369                 spec->data[0] = vid;
 370         }
 371
 372         spec->data[1] = addr[2] << 24 | addr[3] << 16 | addr[4] << 8 | addr[5];
 373         spec->data[2] = addr[0] << 8 | addr[1];
 374         return 0;
 375 }
 376
 377 /**
 378  * efx_filter_set_uc_def - specify matching otherwise-unmatched unicast
 379  * @spec: Specification to initialise
 380  */
 381 int efx_filter_set_uc_def(struct efx_filter_spec *spec)
 382 {
 383         EFX_BUG_ON_PARANOID(!(spec->flags &
 384                               (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX)));
 385
 386         if (spec->type != EFX_FILTER_UNSPEC)
 387                 return -EINVAL;
 388
 389         spec->type = EFX_FILTER_UC_DEF;
 390         memset(spec->data, 0, sizeof(spec->data)); /* ensure equality */
 391         return 0;
 392 }
 393
 394 /**
 395  * efx_filter_set_mc_def - specify matching otherwise-unmatched multicast
 396  * @spec: Specification to initialise
 397  */
 398 int efx_filter_set_mc_def(struct efx_filter_spec *spec)
 399 {
 400         EFX_BUG_ON_PARANOID(!(spec->flags &
 401                               (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX)));
 402
 403         if (spec->type != EFX_FILTER_UNSPEC)
 404                 return -EINVAL;
 405
 406         spec->type = EFX_FILTER_MC_DEF;
 407         memset(spec->data, 0, sizeof(spec->data)); /* ensure equality */
 408         return 0;
 409 }
 410
 411 static void efx_filter_reset_rx_def(struct efx_nic *efx, unsigned filter_idx)
 412 {
 413         struct efx_filter_state *state = efx->filter_state;
 414         struct efx_filter_table *table = &state->table[EFX_FILTER_TABLE_RX_DEF];
 415         struct efx_filter_spec *spec = &table->spec[filter_idx];
 416
 417         /* If there's only one channel then disable RSS for non VF
 418          * traffic, thereby allowing VFs to use RSS when the PF can't.
 419          */
 420         efx_filter_init_rx(spec, EFX_FILTER_PRI_MANUAL,
 421                            efx->n_rx_channels > 1 ? EFX_FILTER_FLAG_RX_RSS : 0,
 422                            0);
 423         spec->type = EFX_FILTER_UC_DEF + filter_idx;
 424         table->used_bitmap[0] |= 1 << filter_idx;
 425 }
 426
 427 int efx_filter_get_eth_local(const struct efx_filter_spec *spec,
 428                              u16 *vid, u8 *addr)
 429 {
 430         switch (spec->type) {
 431         case EFX_FILTER_MAC_WILD:
 432                 *vid = EFX_FILTER_VID_UNSPEC;
 433                 break;
 434         case EFX_FILTER_MAC_FULL:
 435                 *vid = spec->data[0];
 436                 break;
 437         default:
 438                 return -EINVAL;
 439         }
 440
 441         addr[0] = spec->data[2] >> 8;
 442         addr[1] = spec->data[2];
 443         addr[2] = spec->data[1] >> 24;
 444         addr[3] = spec->data[1] >> 16;
 445         addr[4] = spec->data[1] >> 8;
 446         addr[5] = spec->data[1];
 447         return 0;
 448 }
 449
 450 /* Build a filter entry and return its n-tuple key. */
 451 static u32 efx_filter_build(efx_oword_t *filter, struct efx_filter_spec *spec)
 452 {
 453         u32 data3;
 454
 455         switch (efx_filter_spec_table_id(spec)) {
 456         case EFX_FILTER_TABLE_RX_IP: {
 457                 bool is_udp = (spec->type == EFX_FILTER_UDP_FULL ||
 458                                spec->type == EFX_FILTER_UDP_WILD);
 459                 EFX_POPULATE_OWORD_7(
 460                         *filter,
 461                         FRF_BZ_RSS_EN,
 462                         !!(spec->flags & EFX_FILTER_FLAG_RX_RSS),
 463                         FRF_BZ_SCATTER_EN,
 464                         !!(spec->flags & EFX_FILTER_FLAG_RX_SCATTER),
 465                         FRF_BZ_TCP_UDP, is_udp,
 466                         FRF_BZ_RXQ_ID, spec->dmaq_id,
 467                         EFX_DWORD_2, spec->data[2],
 468                         EFX_DWORD_1, spec->data[1],
 469                         EFX_DWORD_0, spec->data[0]);
 470                 data3 = is_udp;
 471                 break;
 472         }
 473
 474         case EFX_FILTER_TABLE_RX_MAC: {
 475                 bool is_wild = spec->type == EFX_FILTER_MAC_WILD;
 476                 EFX_POPULATE_OWORD_7(
 477                         *filter,
 478                         FRF_CZ_RMFT_RSS_EN,
 479                         !!(spec->flags & EFX_FILTER_FLAG_RX_RSS),
 480                         FRF_CZ_RMFT_SCATTER_EN,
 481                         !!(spec->flags & EFX_FILTER_FLAG_RX_SCATTER),
 482                         FRF_CZ_RMFT_RXQ_ID, spec->dmaq_id,
 483                         FRF_CZ_RMFT_WILDCARD_MATCH, is_wild,
 484                         FRF_CZ_RMFT_DEST_MAC_HI, spec->data[2],
 485                         FRF_CZ_RMFT_DEST_MAC_LO, spec->data[1],
 486                         FRF_CZ_RMFT_VLAN_ID, spec->data[0]);
 487                 data3 = is_wild;
 488                 break;
 489         }
 490
 491         case EFX_FILTER_TABLE_TX_MAC: {
 492                 bool is_wild = spec->type == EFX_FILTER_MAC_WILD;
 493                 EFX_POPULATE_OWORD_5(*filter,
 494                                      FRF_CZ_TMFT_TXQ_ID, spec->dmaq_id,
 495                                      FRF_CZ_TMFT_WILDCARD_MATCH, is_wild,
 496                                      FRF_CZ_TMFT_SRC_MAC_HI, spec->data[2],
 497                                      FRF_CZ_TMFT_SRC_MAC_LO, spec->data[1],
 498                                      FRF_CZ_TMFT_VLAN_ID, spec->data[0]);
 499                 data3 = is_wild | spec->dmaq_id << 1;
 500                 break;
 501         }
 502
 503         default:
 504                 BUG();
 505         }
 506
 507         return spec->data[0] ^ spec->data[1] ^ spec->data[2] ^ data3;
 508 }
 509
 510 static bool efx_filter_equal(const struct efx_filter_spec *left,
 511                              const struct efx_filter_spec *right)
 512 {
 513         if (left->type != right->type ||
 514             memcmp(left->data, right->data, sizeof(left->data)))
 515                 return false;
 516
 517         if (left->flags & EFX_FILTER_FLAG_TX &&
 518             left->dmaq_id != right->dmaq_id)
 519                 return false;
 520
 521         return true;
 522 }
 523
 524 /*
 525  * Construct/deconstruct external filter IDs.  At least the RX filter
 526  * IDs must be ordered by matching priority, for RX NFC semantics.
 527  *
 528  * Deconstruction needs to be robust against invalid IDs so that
 529  * efx_filter_remove_id_safe() and efx_filter_get_filter_safe() can
 530  * accept user-provided IDs.
 531  */
 532
 533 #define EFX_FILTER_MATCH_PRI_COUNT      5
 534
 535 static const u8 efx_filter_type_match_pri[EFX_FILTER_TYPE_COUNT] = {
 536         [EFX_FILTER_TCP_FULL]   = 0,
 537         [EFX_FILTER_UDP_FULL]   = 0,
 538         [EFX_FILTER_TCP_WILD]   = 1,
 539         [EFX_FILTER_UDP_WILD]   = 1,
 540         [EFX_FILTER_MAC_FULL]   = 2,
 541         [EFX_FILTER_MAC_WILD]   = 3,
 542         [EFX_FILTER_UC_DEF]     = 4,
 543         [EFX_FILTER_MC_DEF]     = 4,
 544 };
 545
 546 static const enum efx_filter_table_id efx_filter_range_table[] = {
 547         EFX_FILTER_TABLE_RX_IP,         /* RX match pri 0 */
 548         EFX_FILTER_TABLE_RX_IP,
 549         EFX_FILTER_TABLE_RX_MAC,
 550         EFX_FILTER_TABLE_RX_MAC,
 551         EFX_FILTER_TABLE_RX_DEF,        /* RX match pri 4 */
 552         EFX_FILTER_TABLE_COUNT,         /* TX match pri 0; invalid */
 553         EFX_FILTER_TABLE_COUNT,         /* invalid */
 554         EFX_FILTER_TABLE_TX_MAC,
 555         EFX_FILTER_TABLE_TX_MAC,        /* TX match pri 3 */
 556 };
 557
 558 #define EFX_FILTER_INDEX_WIDTH  13
 559 #define EFX_FILTER_INDEX_MASK   ((1 << EFX_FILTER_INDEX_WIDTH) - 1)
 560
 561 static inline u32
 562 efx_filter_make_id(const struct efx_filter_spec *spec, unsigned int index)
 563 {
 564         unsigned int range;
 565
 566         range = efx_filter_type_match_pri[spec->type];
 567         if (!(spec->flags & EFX_FILTER_FLAG_RX))
 568                 range += EFX_FILTER_MATCH_PRI_COUNT;
 569
 570         return range << EFX_FILTER_INDEX_WIDTH | index;
 571 }
 572
 573 static inline enum efx_filter_table_id efx_filter_id_table_id(u32 id)
 574 {
 575         unsigned int range = id >> EFX_FILTER_INDEX_WIDTH;
 576
 577         if (range < ARRAY_SIZE(efx_filter_range_table))
 578                 return efx_filter_range_table[range];
 579         else
 580                 return EFX_FILTER_TABLE_COUNT; /* invalid */
 581 }
 582
 583 static inline unsigned int efx_filter_id_index(u32 id)
 584 {
 585         return id & EFX_FILTER_INDEX_MASK;
 586 }
 587
 588 static inline u8 efx_filter_id_flags(u32 id)
 589 {
 590         unsigned int range = id >> EFX_FILTER_INDEX_WIDTH;
 591
 592         if (range < EFX_FILTER_MATCH_PRI_COUNT)
 593                 return EFX_FILTER_FLAG_RX;
 594         else
 595                 return EFX_FILTER_FLAG_TX;
 596 }
 597
 598 u32 efx_filter_get_rx_id_limit(struct efx_nic *efx)
 599 {
 600         struct efx_filter_state *state = efx->filter_state;
 601         unsigned int range = EFX_FILTER_MATCH_PRI_COUNT - 1;
 602         enum efx_filter_table_id table_id;
 603
 604         do {
 605                 table_id = efx_filter_range_table[range];
 606                 if (state->table[table_id].size != 0)
 607                         return range << EFX_FILTER_INDEX_WIDTH |
 608                                 state->table[table_id].size;
 609         } while (range--);
 610
 611         return 0;
 612 }
 613
 614 /**
 615  * efx_filter_insert_filter - add or replace a filter
 616  * @efx: NIC in which to insert the filter
 617  * @spec: Specification for the filter
 618  * @replace_equal: Flag for whether the specified filter may replace an
 619  *      existing filter with equal priority
 620  *
 621  * On success, return the filter ID.
 622  * On failure, return a negative error code.
 623  *
 624  * If an existing filter has equal match values to the new filter
 625  * spec, then the new filter might replace it, depending on the
 626  * relative priorities.  If the existing filter has lower priority, or
 627  * if @replace_equal is set and it has equal priority, then it is
 628  * replaced.  Otherwise the function fails, returning -%EPERM if
 629  * the existing filter has higher priority or -%EEXIST if it has
 630  * equal priority.
 631  */
 632 s32 efx_filter_insert_filter(struct efx_nic *efx, struct efx_filter_spec *spec,
 633                              bool replace_equal)
 634 {
 635         struct efx_filter_state *state = efx->filter_state;
 636         struct efx_filter_table *table = efx_filter_spec_table(state, spec);
 637         efx_oword_t filter;
 638         int rep_index, ins_index;
 639         unsigned int depth = 0;
 640         int rc;
 641
 642         if (!table || table->size == 0)
 643                 return -EINVAL;
 644
 645         netif_vdbg(efx, hw, efx->net_dev,
 646                    "%s: type %d search_depth=%d", __func__, spec->type,
 647                    table->search_depth[spec->type]);
 648
 649         if (table->id == EFX_FILTER_TABLE_RX_DEF) {
 650                 /* One filter spec per type */
 651                 BUILD_BUG_ON(EFX_FILTER_INDEX_UC_DEF != 0);
 652                 BUILD_BUG_ON(EFX_FILTER_INDEX_MC_DEF !=
 653                              EFX_FILTER_MC_DEF - EFX_FILTER_UC_DEF);
 654                 rep_index = spec->type - EFX_FILTER_INDEX_UC_DEF;
 655                 ins_index = rep_index;
 656
 657                 spin_lock_bh(&state->lock);
 658         } else {
 659                 /* Search concurrently for
 660                  * (1) a filter to be replaced (rep_index): any filter
 661                  *     with the same match values, up to the current
 662                  *     search depth for this type, and
 663                  * (2) the insertion point (ins_index): (1) or any
 664                  *     free slot before it or up to the maximum search
 665                  *     depth for this priority
 666                  * We fail if we cannot find (2).
 667                  *
 668                  * We can stop once either
 669                  * (a) we find (1), in which case we have definitely
 670                  *     found (2) as well; or
 671                  * (b) we have searched exhaustively for (1), and have
 672                  *     either found (2) or searched exhaustively for it
 673                  */
 674                 u32 key = efx_filter_build(&filter, spec);
 675                 unsigned int hash = efx_filter_hash(key);
 676                 unsigned int incr = efx_filter_increment(key);
 677                 unsigned int max_rep_depth = table->search_depth[spec->type];
 678                 unsigned int max_ins_depth =
 679                         spec->priority <= EFX_FILTER_PRI_HINT ?
 680                         FILTER_CTL_SRCH_HINT_MAX : FILTER_CTL_SRCH_MAX;
 681                 unsigned int i = hash & (table->size - 1);
 682
 683                 ins_index = -1;
 684                 depth = 1;
 685
 686                 spin_lock_bh(&state->lock);
 687
 688                 for (;;) {
 689                         if (!test_bit(i, table->used_bitmap)) {
 690                                 if (ins_index < 0)
 691                                         ins_index = i;
 692                         } else if (efx_filter_equal(spec, &table->spec[i])) {
 693                                 /* Case (a) */
 694                                 if (ins_index < 0)
 695                                         ins_index = i;
 696                                 rep_index = i;
 697                                 break;
 698                         }
 699
 700                         if (depth >= max_rep_depth &&
 701                             (ins_index >= 0 || depth >= max_ins_depth)) {
 702                                 /* Case (b) */
 703                                 if (ins_index < 0) {
 704                                         rc = -EBUSY;
 705                                         goto out;
 706                                 }
 707                                 rep_index = -1;
 708                                 break;
 709                         }
 710
 711                         i = (i + incr) & (table->size - 1);
 712                         ++depth;
 713                 }
 714         }
 715
 716         /* If we found a filter to be replaced, check whether we
 717          * should do so
 718          */
 719         if (rep_index >= 0) {
 720                 struct efx_filter_spec *saved_spec = &table->spec[rep_index];
 721
 722                 if (spec->priority == saved_spec->priority && !replace_equal) {
 723                         rc = -EEXIST;
 724                         goto out;
 725                 }
 726                 if (spec->priority < saved_spec->priority) {
 727                         rc = -EPERM;
 728                         goto out;
 729                 }
 730         }
 731
 732         /* Insert the filter */
 733         if (ins_index != rep_index) {
 734                 __set_bit(ins_index, table->used_bitmap);
 735                 ++table->used;
 736         }
 737         table->spec[ins_index] = *spec;
 738
 739         if (table->id == EFX_FILTER_TABLE_RX_DEF) {
 740                 efx_filter_push_rx_config(efx);
 741         } else {
 742                 if (table->search_depth[spec->type] < depth) {
 743                         table->search_depth[spec->type] = depth;
 744                         if (spec->flags & EFX_FILTER_FLAG_TX)
 745                                 efx_filter_push_tx_limits(efx);
 746                         else
 747                                 efx_filter_push_rx_config(efx);
 748                 }
 749
 750                 efx_writeo(efx, &filter,
 751                            table->offset + table->step * ins_index);
 752
 753                 /* If we were able to replace a filter by inserting
 754                  * at a lower depth, clear the replaced filter
 755                  */
 756                 if (ins_index != rep_index && rep_index >= 0)
 757                         efx_filter_table_clear_entry(efx, table, rep_index);
 758         }
 759
 760         netif_vdbg(efx, hw, efx->net_dev,
 761                    "%s: filter type %d index %d rxq %u set",
 762                    __func__, spec->type, ins_index, spec->dmaq_id);
 763         rc = efx_filter_make_id(spec, ins_index);
 764
 765 out:
 766         spin_unlock_bh(&state->lock);
 767         return rc;
 768 }
 769
 770 static void efx_filter_table_clear_entry(struct efx_nic *efx,
 771                                          struct efx_filter_table *table,
 772                                          unsigned int filter_idx)
 773 {
 774         static efx_oword_t filter;
 775
 776         if (table->id == EFX_FILTER_TABLE_RX_DEF) {
 777                 /* RX default filters must always exist */
 778                 efx_filter_reset_rx_def(efx, filter_idx);
 779                 efx_filter_push_rx_config(efx);
 780         } else if (test_bit(filter_idx, table->used_bitmap)) {
 781                 __clear_bit(filter_idx, table->used_bitmap);
 782                 --table->used;
 783                 memset(&table->spec[filter_idx], 0, sizeof(table->spec[0]));
 784
 785                 efx_writeo(efx, &filter,
 786                            table->offset + table->step * filter_idx);
 787         }
 788 }
 789
 790 /**
 791  * efx_filter_remove_id_safe - remove a filter by ID, carefully
 792  * @efx: NIC from which to remove the filter
 793  * @priority: Priority of filter, as passed to @efx_filter_insert_filter
 794  * @filter_id: ID of filter, as returned by @efx_filter_insert_filter
 795  *
 796  * This function will range-check @filter_id, so it is safe to call
 797  * with a value passed from userland.
 798  */
 799 int efx_filter_remove_id_safe(struct efx_nic *efx,
 800                               enum efx_filter_priority priority,
 801                               u32 filter_id)
 802 {
 803         struct efx_filter_state *state = efx->filter_state;
 804         enum efx_filter_table_id table_id;
 805         struct efx_filter_table *table;
 806         unsigned int filter_idx;
 807         struct efx_filter_spec *spec;
 808         u8 filter_flags;
 809         int rc;
 810
 811         table_id = efx_filter_id_table_id(filter_id);
 812         if ((unsigned int)table_id >= EFX_FILTER_TABLE_COUNT)
 813                 return -ENOENT;
 814         table = &state->table[table_id];
 815
 816         filter_idx = efx_filter_id_index(filter_id);
 817         if (filter_idx >= table->size)
 818                 return -ENOENT;
 819         spec = &table->spec[filter_idx];
 820
 821         filter_flags = efx_filter_id_flags(filter_id);
 822
 823         spin_lock_bh(&state->lock);
 824
 825         if (test_bit(filter_idx, table->used_bitmap) &&
 826             spec->priority == priority) {
 827                 efx_filter_table_clear_entry(efx, table, filter_idx);
 828                 if (table->used == 0)
 829                         efx_filter_table_reset_search_depth(table);
 830                 rc = 0;
 831         } else {
 832                 rc = -ENOENT;
 833         }
 834
 835         spin_unlock_bh(&state->lock);
 836
 837         return rc;
 838 }
 839
 840 /**
 841  * efx_filter_get_filter_safe - retrieve a filter by ID, carefully
 842  * @efx: NIC from which to remove the filter
 843  * @priority: Priority of filter, as passed to @efx_filter_insert_filter
 844  * @filter_id: ID of filter, as returned by @efx_filter_insert_filter
 845  * @spec: Buffer in which to store filter specification
 846  *
 847  * This function will range-check @filter_id, so it is safe to call
 848  * with a value passed from userland.
 849  */
 850 int efx_filter_get_filter_safe(struct efx_nic *efx,
 851                                enum efx_filter_priority priority,
 852                                u32 filter_id, struct efx_filter_spec *spec_buf)
 853 {
 854         struct efx_filter_state *state = efx->filter_state;
 855         enum efx_filter_table_id table_id;
 856         struct efx_filter_table *table;
 857         struct efx_filter_spec *spec;
 858         unsigned int filter_idx;
 859         u8 filter_flags;
 860         int rc;
 861
 862         table_id = efx_filter_id_table_id(filter_id);
 863         if ((unsigned int)table_id >= EFX_FILTER_TABLE_COUNT)
 864                 return -ENOENT;
 865         table = &state->table[table_id];
 866
 867         filter_idx = efx_filter_id_index(filter_id);
 868         if (filter_idx >= table->size)
 869                 return -ENOENT;
 870         spec = &table->spec[filter_idx];
 871
 872         filter_flags = efx_filter_id_flags(filter_id);
 873
 874         spin_lock_bh(&state->lock);
 875
 876         if (test_bit(filter_idx, table->used_bitmap) &&
 877             spec->priority == priority) {
 878                 *spec_buf = *spec;
 879                 rc = 0;
 880         } else {
 881                 rc = -ENOENT;
 882         }
 883
 884         spin_unlock_bh(&state->lock);
 885
 886         return rc;
 887 }
 888
 889 static void efx_filter_table_clear(struct efx_nic *efx,
 890                                    enum efx_filter_table_id table_id,
 891                                    enum efx_filter_priority priority)
 892 {
 893         struct efx_filter_state *state = efx->filter_state;
 894         struct efx_filter_table *table = &state->table[table_id];
 895         unsigned int filter_idx;
 896
 897         spin_lock_bh(&state->lock);
 898
 899         for (filter_idx = 0; filter_idx < table->size; ++filter_idx)
 900                 if (table->spec[filter_idx].priority <= priority)
 901                         efx_filter_table_clear_entry(efx, table, filter_idx);
 902         if (table->used == 0)
 903                 efx_filter_table_reset_search_depth(table);
 904
 905         spin_unlock_bh(&state->lock);
 906 }
 907
 908 /**
 909  * efx_filter_clear_rx - remove RX filters by priority
 910  * @efx: NIC from which to remove the filters
 911  * @priority: Maximum priority to remove
 912  */
 913 void efx_filter_clear_rx(struct efx_nic *efx, enum efx_filter_priority priority)
 914 {
 915         efx_filter_table_clear(efx, EFX_FILTER_TABLE_RX_IP, priority);
 916         efx_filter_table_clear(efx, EFX_FILTER_TABLE_RX_MAC, priority);
 917 }
 918
 919 u32 efx_filter_count_rx_used(struct efx_nic *efx,
 920                              enum efx_filter_priority priority)
 921 {
 922         struct efx_filter_state *state = efx->filter_state;
 923         enum efx_filter_table_id table_id;
 924         struct efx_filter_table *table;
 925         unsigned int filter_idx;
 926         u32 count = 0;
 927
 928         spin_lock_bh(&state->lock);
 929
 930         for (table_id = EFX_FILTER_TABLE_RX_IP;
 931              table_id <= EFX_FILTER_TABLE_RX_DEF;
 932              table_id++) {
 933                 table = &state->table[table_id];
 934                 for (filter_idx = 0; filter_idx < table->size; filter_idx++) {
 935                         if (test_bit(filter_idx, table->used_bitmap) &&
 936                             table->spec[filter_idx].priority == priority)
 937                                 ++count;
 938                 }
 939         }
 940
 941         spin_unlock_bh(&state->lock);
 942
 943         return count;
 944 }
 945
 946 s32 efx_filter_get_rx_ids(struct efx_nic *efx,
 947                           enum efx_filter_priority priority,
 948                           u32 *buf, u32 size)
 949 {
 950         struct efx_filter_state *state = efx->filter_state;
 951         enum efx_filter_table_id table_id;
 952         struct efx_filter_table *table;
 953         unsigned int filter_idx;
 954         s32 count = 0;
 955
 956         spin_lock_bh(&state->lock);
 957
 958         for (table_id = EFX_FILTER_TABLE_RX_IP;
 959              table_id <= EFX_FILTER_TABLE_RX_DEF;
 960              table_id++) {
 961                 table = &state->table[table_id];
 962                 for (filter_idx = 0; filter_idx < table->size; filter_idx++) {
 963                         if (test_bit(filter_idx, table->used_bitmap) &&
 964                             table->spec[filter_idx].priority == priority) {
 965                                 if (count == size) {
 966                                         count = -EMSGSIZE;
 967                                         goto out;
 968                                 }
 969                                 buf[count++] = efx_filter_make_id(
 970                                         &table->spec[filter_idx], filter_idx);
 971                         }
 972                 }
 973         }
 974 out:
 975         spin_unlock_bh(&state->lock);
 976
 977         return count;
 978 }
 979
 980 /* Restore filter stater after reset */
 981 void efx_restore_filters(struct efx_nic *efx)
 982 {
 983         struct efx_filter_state *state = efx->filter_state;
 984         enum efx_filter_table_id table_id;
 985         struct efx_filter_table *table;
 986         efx_oword_t filter;
 987         unsigned int filter_idx;
 988
 989         spin_lock_bh(&state->lock);
 990
 991         for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) {
 992                 table = &state->table[table_id];
 993
 994                 /* Check whether this is a regular register table */
 995                 if (table->step == 0)
 996                         continue;
 997
 998                 for (filter_idx = 0; filter_idx < table->size; filter_idx++) {
 999                         if (!test_bit(filter_idx, table->used_bitmap))
1000                                 continue;
1001                         efx_filter_build(&filter, &table->spec[filter_idx]);
1002                         efx_writeo(efx, &filter,
1003                                    table->offset + table->step * filter_idx);
1004                 }
1005         }
1006
1007         efx_filter_push_rx_config(efx);
1008         efx_filter_push_tx_limits(efx);
1009
1010         spin_unlock_bh(&state->lock);
1011 }
1012
1013 int efx_probe_filters(struct efx_nic *efx)
1014 {
1015         struct efx_filter_state *state;
1016         struct efx_filter_table *table;
1017         unsigned table_id;
1018
1019         state = kzalloc(sizeof(*efx->filter_state), GFP_KERNEL);
1020         if (!state)
1021                 return -ENOMEM;
1022         efx->filter_state = state;
1023
1024         spin_lock_init(&state->lock);
1025
1026         if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) {
1027 #ifdef CONFIG_RFS_ACCEL
1028                 state->rps_flow_id = kcalloc(FR_BZ_RX_FILTER_TBL0_ROWS,
1029                                              sizeof(*state->rps_flow_id),
1030                                              GFP_KERNEL);
1031                 if (!state->rps_flow_id)
1032                         goto fail;
1033 #endif
1034                 table = &state->table[EFX_FILTER_TABLE_RX_IP];
1035                 table->id = EFX_FILTER_TABLE_RX_IP;
1036                 table->offset = FR_BZ_RX_FILTER_TBL0;
1037                 table->size = FR_BZ_RX_FILTER_TBL0_ROWS;
1038                 table->step = FR_BZ_RX_FILTER_TBL0_STEP;
1039         }
1040
1041         if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0) {
1042                 table = &state->table[EFX_FILTER_TABLE_RX_MAC];
1043                 table->id = EFX_FILTER_TABLE_RX_MAC;
1044                 table->offset = FR_CZ_RX_MAC_FILTER_TBL0;
1045                 table->size = FR_CZ_RX_MAC_FILTER_TBL0_ROWS;
1046                 table->step = FR_CZ_RX_MAC_FILTER_TBL0_STEP;
1047
1048                 table = &state->table[EFX_FILTER_TABLE_RX_DEF];
1049                 table->id = EFX_FILTER_TABLE_RX_DEF;
1050                 table->size = EFX_FILTER_SIZE_RX_DEF;
1051
1052                 table = &state->table[EFX_FILTER_TABLE_TX_MAC];
1053                 table->id = EFX_FILTER_TABLE_TX_MAC;
1054                 table->offset = FR_CZ_TX_MAC_FILTER_TBL0;
1055                 table->size = FR_CZ_TX_MAC_FILTER_TBL0_ROWS;
1056                 table->step = FR_CZ_TX_MAC_FILTER_TBL0_STEP;
1057         }
1058
1059         for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) {
1060                 table = &state->table[table_id];
1061                 if (table->size == 0)
1062                         continue;
1063                 table->used_bitmap = kcalloc(BITS_TO_LONGS(table->size),
1064                                              sizeof(unsigned long),
1065                                              GFP_KERNEL);
1066                 if (!table->used_bitmap)
1067                         goto fail;
1068                 table->spec = vzalloc(table->size * sizeof(*table->spec));
1069                 if (!table->spec)
1070                         goto fail;
1071         }
1072
1073         if (state->table[EFX_FILTER_TABLE_RX_DEF].size) {
1074                 /* RX default filters must always exist */
1075                 unsigned i;
1076                 for (i = 0; i < EFX_FILTER_SIZE_RX_DEF; i++)
1077                         efx_filter_reset_rx_def(efx, i);
1078         }
1079
1080         efx_filter_push_rx_config(efx);
1081
1082         return 0;
1083
1084 fail:
1085         efx_remove_filters(efx);
1086         return -ENOMEM;
1087 }
1088
1089 void efx_remove_filters(struct efx_nic *efx)
1090 {
1091         struct efx_filter_state *state = efx->filter_state;
1092         enum efx_filter_table_id table_id;
1093
1094         for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) {
1095                 kfree(state->table[table_id].used_bitmap);
1096                 vfree(state->table[table_id].spec);
1097         }
1098 #ifdef CONFIG_RFS_ACCEL
1099         kfree(state->rps_flow_id);
1100 #endif
1101         kfree(state);
1102 }
1103
1104 #ifdef CONFIG_RFS_ACCEL
1105
1106 int efx_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb,
1107                    u16 rxq_index, u32 flow_id)
1108 {
1109         struct efx_nic *efx = netdev_priv(net_dev);
1110         struct efx_channel *channel;
1111         struct efx_filter_state *state = efx->filter_state;
1112         struct efx_filter_spec spec;
1113         const struct iphdr *ip;
1114         const __be16 *ports;
1115         int nhoff;
1116         int rc;
1117
1118         nhoff = skb_network_offset(skb);
1119
1120         if (skb->protocol != htons(ETH_P_IP))
1121                 return -EPROTONOSUPPORT;
1122
1123         /* RFS must validate the IP header length before calling us */
1124         EFX_BUG_ON_PARANOID(skb_headlen(skb) < nhoff + sizeof(*ip));
1125         ip = (const struct iphdr *)(skb->data + nhoff);
1126         if (ip_is_fragment(ip))
1127                 return -EPROTONOSUPPORT;
1128         EFX_BUG_ON_PARANOID(skb_headlen(skb) < nhoff + 4 * ip->ihl + 4);
1129         ports = (const __be16 *)(skb->data + nhoff + 4 * ip->ihl);
1130
1131         efx_filter_init_rx(&spec, EFX_FILTER_PRI_HINT, 0, rxq_index);
1132         rc = efx_filter_set_ipv4_full(&spec, ip->protocol,
1133                                       ip->daddr, ports[1], ip->saddr, ports[0]);
1134         if (rc)
1135                 return rc;
1136
1137         rc = efx_filter_insert_filter(efx, &spec, true);
1138         if (rc < 0)
1139                 return rc;
1140
1141         /* Remember this so we can check whether to expire the filter later */
1142         state->rps_flow_id[rc] = flow_id;
1143         channel = efx_get_channel(efx, skb_get_rx_queue(skb));
1144         ++channel->rfs_filters_added;
1145
1146         netif_info(efx, rx_status, efx->net_dev,
1147                    "steering %s %pI4:%u:%pI4:%u to queue %u [flow %u filter %d]\n",
1148                    (ip->protocol == IPPROTO_TCP) ? "TCP" : "UDP",
1149                    &ip->saddr, ntohs(ports[0]), &ip->daddr, ntohs(ports[1]),
1150                    rxq_index, flow_id, rc);
1151
1152         return rc;
1153 }
1154
1155 bool __efx_filter_rfs_expire(struct efx_nic *efx, unsigned quota)
1156 {
1157         struct efx_filter_state *state = efx->filter_state;
1158         struct efx_filter_table *table = &state->table[EFX_FILTER_TABLE_RX_IP];
1159         unsigned mask = table->size - 1;
1160         unsigned index;
1161         unsigned stop;
1162
1163         if (!spin_trylock_bh(&state->lock))
1164                 return false;
1165
1166         index = state->rps_expire_index;
1167         stop = (index + quota) & mask;
1168
1169         while (index != stop) {
1170                 if (test_bit(index, table->used_bitmap) &&
1171                     table->spec[index].priority == EFX_FILTER_PRI_HINT &&
1172                     rps_may_expire_flow(efx->net_dev,
1173                                         table->spec[index].dmaq_id,
1174                                         state->rps_flow_id[index], index)) {
1175                         netif_info(efx, rx_status, efx->net_dev,
1176                                    "expiring filter %d [flow %u]\n",
1177                                    index, state->rps_flow_id[index]);
1178                         efx_filter_table_clear_entry(efx, table, index);
1179                 }
1180                 index = (index + 1) & mask;
1181         }
1182
1183         state->rps_expire_index = stop;
1184         if (table->used == 0)
1185                 efx_filter_table_reset_search_depth(table);
1186
1187         spin_unlock_bh(&state->lock);
1188         return true;
1189 }
1190
1191 #endif /* CONFIG_RFS_ACCEL */