drivers/net/ethernet/sfc/filter.c

   1 /****************************************************************************
   2  * Driver for Solarflare Solarstorm network controllers and boards
   3  * Copyright 2005-2010 Solarflare Communications Inc.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 as published
   7  * by the Free Software Foundation, incorporated herein by reference.
   8  */
   9
  10 #include <linux/in.h>
  11 #include <net/ip.h>
  12 #include "efx.h"
  13 #include "filter.h"
  14 #include "io.h"
  15 #include "nic.h"
  16 #include "regs.h"
  17
  18 /* "Fudge factors" - difference between programmed value and actual depth.
  19  * Due to pipelined implementation we need to program H/W with a value that
  20  * is larger than the hop limit we want.
  21  */
  22 #define FILTER_CTL_SRCH_FUDGE_WILD 3
  23 #define FILTER_CTL_SRCH_FUDGE_FULL 1
  24
  25 /* Hard maximum hop limit.  Hardware will time-out beyond 200-something.
  26  * We also need to avoid infinite loops in efx_filter_search() when the
  27  * table is full.
  28  */
  29 #define FILTER_CTL_SRCH_MAX 200
  30
  31 /* Don't try very hard to find space for performance hints, as this is
  32  * counter-productive. */
  33 #define FILTER_CTL_SRCH_HINT_MAX 5
  34
  35 enum efx_filter_table_id {
  36         EFX_FILTER_TABLE_RX_IP = 0,
  37         EFX_FILTER_TABLE_RX_MAC,
  38         EFX_FILTER_TABLE_RX_DEF,
  39         EFX_FILTER_TABLE_TX_MAC,
  40         EFX_FILTER_TABLE_COUNT,
  41 };
  42
  43 enum efx_filter_index {
  44         EFX_FILTER_INDEX_UC_DEF,
  45         EFX_FILTER_INDEX_MC_DEF,
  46         EFX_FILTER_SIZE_RX_DEF,
  47 };
  48
  49 struct efx_filter_table {
  50         enum efx_filter_table_id id;
  51         u32             offset;         /* address of table relative to BAR */
  52         unsigned        size;           /* number of entries */
  53         unsigned        step;           /* step between entries */
  54         unsigned        used;           /* number currently used */
  55         unsigned long   *used_bitmap;
  56         struct efx_filter_spec *spec;
  57         unsigned        search_depth[EFX_FILTER_TYPE_COUNT];
  58 };
  59
  60 struct efx_filter_state {
  61         spinlock_t      lock;
  62         struct efx_filter_table table[EFX_FILTER_TABLE_COUNT];
  63 #ifdef CONFIG_RFS_ACCEL
  64         u32             *rps_flow_id;
  65         unsigned        rps_expire_index;
  66 #endif
  67 };
  68
  69 static void efx_filter_table_clear_entry(struct efx_nic *efx,
  70                                          struct efx_filter_table *table,
  71                                          unsigned int filter_idx);
  72
  73 /* The filter hash function is LFSR polynomial x^16 + x^3 + 1 of a 32-bit
  74  * key derived from the n-tuple.  The initial LFSR state is 0xffff. */
  75 static u16 efx_filter_hash(u32 key)
  76 {
  77         u16 tmp;
  78
  79         /* First 16 rounds */
  80         tmp = 0x1fff ^ key >> 16;
  81         tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
  82         tmp = tmp ^ tmp >> 9;
  83         /* Last 16 rounds */
  84         tmp = tmp ^ tmp << 13 ^ key;
  85         tmp = tmp ^ tmp >> 3 ^ tmp >> 6;
  86         return tmp ^ tmp >> 9;
  87 }
  88
  89 /* To allow for hash collisions, filter search continues at these
  90  * increments from the first possible entry selected by the hash. */
  91 static u16 efx_filter_increment(u32 key)
  92 {
  93         return key * 2 - 1;
  94 }
  95
  96 static enum efx_filter_table_id
  97 efx_filter_spec_table_id(const struct efx_filter_spec *spec)
  98 {
  99         BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_TCP_FULL >> 2));
 100         BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_TCP_WILD >> 2));
 101         BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_UDP_FULL >> 2));
 102         BUILD_BUG_ON(EFX_FILTER_TABLE_RX_IP != (EFX_FILTER_UDP_WILD >> 2));
 103         BUILD_BUG_ON(EFX_FILTER_TABLE_RX_MAC != (EFX_FILTER_MAC_FULL >> 2));
 104         BUILD_BUG_ON(EFX_FILTER_TABLE_RX_MAC != (EFX_FILTER_MAC_WILD >> 2));
 105         BUILD_BUG_ON(EFX_FILTER_TABLE_TX_MAC != EFX_FILTER_TABLE_RX_MAC + 2);
 106         EFX_BUG_ON_PARANOID(spec->type == EFX_FILTER_UNSPEC);
 107         return (spec->type >> 2) + ((spec->flags & EFX_FILTER_FLAG_TX) ? 2 : 0);
 108 }
 109
 110 static struct efx_filter_table *
 111 efx_filter_spec_table(struct efx_filter_state *state,
 112                       const struct efx_filter_spec *spec)
 113 {
 114         if (spec->type == EFX_FILTER_UNSPEC)
 115                 return NULL;
 116         else
 117                 return &state->table[efx_filter_spec_table_id(spec)];
 118 }
 119
 120 static void efx_filter_table_reset_search_depth(struct efx_filter_table *table)
 121 {
 122         memset(table->search_depth, 0, sizeof(table->search_depth));
 123 }
 124
 125 static void efx_filter_push_rx_config(struct efx_nic *efx)
 126 {
 127         struct efx_filter_state *state = efx->filter_state;
 128         struct efx_filter_table *table;
 129         efx_oword_t filter_ctl;
 130
 131         efx_reado(efx, &filter_ctl, FR_BZ_RX_FILTER_CTL);
 132
 133         table = &state->table[EFX_FILTER_TABLE_RX_IP];
 134         EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_TCP_FULL_SRCH_LIMIT,
 135                             table->search_depth[EFX_FILTER_TCP_FULL] +
 136                             FILTER_CTL_SRCH_FUDGE_FULL);
 137         EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_TCP_WILD_SRCH_LIMIT,
 138                             table->search_depth[EFX_FILTER_TCP_WILD] +
 139                             FILTER_CTL_SRCH_FUDGE_WILD);
 140         EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_UDP_FULL_SRCH_LIMIT,
 141                             table->search_depth[EFX_FILTER_UDP_FULL] +
 142                             FILTER_CTL_SRCH_FUDGE_FULL);
 143         EFX_SET_OWORD_FIELD(filter_ctl, FRF_BZ_UDP_WILD_SRCH_LIMIT,
 144                             table->search_depth[EFX_FILTER_UDP_WILD] +
 145                             FILTER_CTL_SRCH_FUDGE_WILD);
 146
 147         table = &state->table[EFX_FILTER_TABLE_RX_MAC];
 148         if (table->size) {
 149                 EFX_SET_OWORD_FIELD(
 150                         filter_ctl, FRF_CZ_ETHERNET_FULL_SEARCH_LIMIT,
 151                         table->search_depth[EFX_FILTER_MAC_FULL] +
 152                         FILTER_CTL_SRCH_FUDGE_FULL);
 153                 EFX_SET_OWORD_FIELD(
 154                         filter_ctl, FRF_CZ_ETHERNET_WILDCARD_SEARCH_LIMIT,
 155                         table->search_depth[EFX_FILTER_MAC_WILD] +
 156                         FILTER_CTL_SRCH_FUDGE_WILD);
 157         }
 158
 159         table = &state->table[EFX_FILTER_TABLE_RX_DEF];
 160         if (table->size) {
 161                 EFX_SET_OWORD_FIELD(
 162                         filter_ctl, FRF_CZ_UNICAST_NOMATCH_Q_ID,
 163                         table->spec[EFX_FILTER_INDEX_UC_DEF].dmaq_id);
 164                 EFX_SET_OWORD_FIELD(
 165                         filter_ctl, FRF_CZ_UNICAST_NOMATCH_RSS_ENABLED,
 166                         !!(table->spec[EFX_FILTER_INDEX_UC_DEF].flags &
 167                            EFX_FILTER_FLAG_RX_RSS));
 168                 EFX_SET_OWORD_FIELD(
 169                         filter_ctl, FRF_CZ_MULTICAST_NOMATCH_Q_ID,
 170                         table->spec[EFX_FILTER_INDEX_MC_DEF].dmaq_id);
 171                 EFX_SET_OWORD_FIELD(
 172                         filter_ctl, FRF_CZ_MULTICAST_NOMATCH_RSS_ENABLED,
 173                         !!(table->spec[EFX_FILTER_INDEX_MC_DEF].flags &
 174                            EFX_FILTER_FLAG_RX_RSS));
 175         }
 176
 177         efx_writeo(efx, &filter_ctl, FR_BZ_RX_FILTER_CTL);
 178 }
 179
 180 static void efx_filter_push_tx_limits(struct efx_nic *efx)
 181 {
 182         struct efx_filter_state *state = efx->filter_state;
 183         struct efx_filter_table *table;
 184         efx_oword_t tx_cfg;
 185
 186         efx_reado(efx, &tx_cfg, FR_AZ_TX_CFG);
 187
 188         table = &state->table[EFX_FILTER_TABLE_TX_MAC];
 189         if (table->size) {
 190                 EFX_SET_OWORD_FIELD(
 191                         tx_cfg, FRF_CZ_TX_ETH_FILTER_FULL_SEARCH_RANGE,
 192                         table->search_depth[EFX_FILTER_MAC_FULL] +
 193                         FILTER_CTL_SRCH_FUDGE_FULL);
 194                 EFX_SET_OWORD_FIELD(
 195                         tx_cfg, FRF_CZ_TX_ETH_FILTER_WILD_SEARCH_RANGE,
 196                         table->search_depth[EFX_FILTER_MAC_WILD] +
 197                         FILTER_CTL_SRCH_FUDGE_WILD);
 198         }
 199
 200         efx_writeo(efx, &tx_cfg, FR_AZ_TX_CFG);
 201 }
 202
 203 static inline void __efx_filter_set_ipv4(struct efx_filter_spec *spec,
 204                                          __be32 host1, __be16 port1,
 205                                          __be32 host2, __be16 port2)
 206 {
 207         spec->data[0] = ntohl(host1) << 16 | ntohs(port1);
 208         spec->data[1] = ntohs(port2) << 16 | ntohl(host1) >> 16;
 209         spec->data[2] = ntohl(host2);
 210 }
 211
 212 static inline void __efx_filter_get_ipv4(const struct efx_filter_spec *spec,
 213                                          __be32 *host1, __be16 *port1,
 214                                          __be32 *host2, __be16 *port2)
 215 {
 216         *host1 = htonl(spec->data[0] >> 16 | spec->data[1] << 16);
 217         *port1 = htons(spec->data[0]);
 218         *host2 = htonl(spec->data[2]);
 219         *port2 = htons(spec->data[1] >> 16);
 220 }
 221
 222 /**
 223  * efx_filter_set_ipv4_local - specify IPv4 host, transport protocol and port
 224  * @spec: Specification to initialise
 225  * @proto: Transport layer protocol number
 226  * @host: Local host address (network byte order)
 227  * @port: Local port (network byte order)
 228  */
 229 int efx_filter_set_ipv4_local(struct efx_filter_spec *spec, u8 proto,
 230                               __be32 host, __be16 port)
 231 {
 232         __be32 host1;
 233         __be16 port1;
 234
 235         EFX_BUG_ON_PARANOID(!(spec->flags & EFX_FILTER_FLAG_RX));
 236
 237         /* This cannot currently be combined with other filtering */
 238         if (spec->type != EFX_FILTER_UNSPEC)
 239                 return -EPROTONOSUPPORT;
 240
 241         if (port == 0)
 242                 return -EINVAL;
 243
 244         switch (proto) {
 245         case IPPROTO_TCP:
 246                 spec->type = EFX_FILTER_TCP_WILD;
 247                 break;
 248         case IPPROTO_UDP:
 249                 spec->type = EFX_FILTER_UDP_WILD;
 250                 break;
 251         default:
 252                 return -EPROTONOSUPPORT;
 253         }
 254
 255         /* Filter is constructed in terms of source and destination,
 256          * with the odd wrinkle that the ports are swapped in a UDP
 257          * wildcard filter.  We need to convert from local and remote
 258          * (= zero for wildcard) addresses.
 259          */
 260         host1 = 0;
 261         if (proto != IPPROTO_UDP) {
 262                 port1 = 0;
 263         } else {
 264                 port1 = port;
 265                 port = 0;
 266         }
 267
 268         __efx_filter_set_ipv4(spec, host1, port1, host, port);
 269         return 0;
 270 }
 271
 272 int efx_filter_get_ipv4_local(const struct efx_filter_spec *spec,
 273                               u8 *proto, __be32 *host, __be16 *port)
 274 {
 275         __be32 host1;
 276         __be16 port1;
 277
 278         switch (spec->type) {
 279         case EFX_FILTER_TCP_WILD:
 280                 *proto = IPPROTO_TCP;
 281                 __efx_filter_get_ipv4(spec, &host1, &port1, host, port);
 282                 return 0;
 283         case EFX_FILTER_UDP_WILD:
 284                 *proto = IPPROTO_UDP;
 285                 __efx_filter_get_ipv4(spec, &host1, port, host, &port1);
 286                 return 0;
 287         default:
 288                 return -EINVAL;
 289         }
 290 }
 291
 292 /**
 293  * efx_filter_set_ipv4_full - specify IPv4 hosts, transport protocol and ports
 294  * @spec: Specification to initialise
 295  * @proto: Transport layer protocol number
 296  * @host: Local host address (network byte order)
 297  * @port: Local port (network byte order)
 298  * @rhost: Remote host address (network byte order)
 299  * @rport: Remote port (network byte order)
 300  */
 301 int efx_filter_set_ipv4_full(struct efx_filter_spec *spec, u8 proto,
 302                              __be32 host, __be16 port,
 303                              __be32 rhost, __be16 rport)
 304 {
 305         EFX_BUG_ON_PARANOID(!(spec->flags & EFX_FILTER_FLAG_RX));
 306
 307         /* This cannot currently be combined with other filtering */
 308         if (spec->type != EFX_FILTER_UNSPEC)
 309                 return -EPROTONOSUPPORT;
 310
 311         if (port == 0 || rport == 0)
 312                 return -EINVAL;
 313
 314         switch (proto) {
 315         case IPPROTO_TCP:
 316                 spec->type = EFX_FILTER_TCP_FULL;
 317                 break;
 318         case IPPROTO_UDP:
 319                 spec->type = EFX_FILTER_UDP_FULL;
 320                 break;
 321         default:
 322                 return -EPROTONOSUPPORT;
 323         }
 324
 325         __efx_filter_set_ipv4(spec, rhost, rport, host, port);
 326         return 0;
 327 }
 328
 329 int efx_filter_get_ipv4_full(const struct efx_filter_spec *spec,
 330                              u8 *proto, __be32 *host, __be16 *port,
 331                              __be32 *rhost, __be16 *rport)
 332 {
 333         switch (spec->type) {
 334         case EFX_FILTER_TCP_FULL:
 335                 *proto = IPPROTO_TCP;
 336                 break;
 337         case EFX_FILTER_UDP_FULL:
 338                 *proto = IPPROTO_UDP;
 339                 break;
 340         default:
 341                 return -EINVAL;
 342         }
 343
 344         __efx_filter_get_ipv4(spec, rhost, rport, host, port);
 345         return 0;
 346 }
 347
 348 /**
 349  * efx_filter_set_eth_local - specify local Ethernet address and optional VID
 350  * @spec: Specification to initialise
 351  * @vid: VLAN ID to match, or %EFX_FILTER_VID_UNSPEC
 352  * @addr: Local Ethernet MAC address
 353  */
 354 int efx_filter_set_eth_local(struct efx_filter_spec *spec,
 355                              u16 vid, const u8 *addr)
 356 {
 357         EFX_BUG_ON_PARANOID(!(spec->flags &
 358                               (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX)));
 359
 360         /* This cannot currently be combined with other filtering */
 361         if (spec->type != EFX_FILTER_UNSPEC)
 362                 return -EPROTONOSUPPORT;
 363
 364         if (vid == EFX_FILTER_VID_UNSPEC) {
 365                 spec->type = EFX_FILTER_MAC_WILD;
 366                 spec->data[0] = 0;
 367         } else {
 368                 spec->type = EFX_FILTER_MAC_FULL;
 369                 spec->data[0] = vid;
 370         }
 371
 372         spec->data[1] = addr[2] << 24 | addr[3] << 16 | addr[4] << 8 | addr[5];
 373         spec->data[2] = addr[0] << 8 | addr[1];
 374         return 0;
 375 }
 376
 377 /**
 378  * efx_filter_set_uc_def - specify matching otherwise-unmatched unicast
 379  * @spec: Specification to initialise
 380  */
 381 int efx_filter_set_uc_def(struct efx_filter_spec *spec)
 382 {
 383         EFX_BUG_ON_PARANOID(!(spec->flags &
 384                               (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX)));
 385
 386         if (spec->type != EFX_FILTER_UNSPEC)
 387                 return -EINVAL;
 388
 389         spec->type = EFX_FILTER_UC_DEF;
 390         memset(spec->data, 0, sizeof(spec->data)); /* ensure equality */
 391         return 0;
 392 }
 393
 394 /**
 395  * efx_filter_set_mc_def - specify matching otherwise-unmatched multicast
 396  * @spec: Specification to initialise
 397  */
 398 int efx_filter_set_mc_def(struct efx_filter_spec *spec)
 399 {
 400         EFX_BUG_ON_PARANOID(!(spec->flags &
 401                               (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX)));
 402
 403         if (spec->type != EFX_FILTER_UNSPEC)
 404                 return -EINVAL;
 405
 406         spec->type = EFX_FILTER_MC_DEF;
 407         memset(spec->data, 0, sizeof(spec->data)); /* ensure equality */
 408         return 0;
 409 }
 410
 411 static void efx_filter_reset_rx_def(struct efx_nic *efx, unsigned filter_idx)
 412 {
 413         struct efx_filter_state *state = efx->filter_state;
 414         struct efx_filter_table *table = &state->table[EFX_FILTER_TABLE_RX_DEF];
 415         struct efx_filter_spec *spec = &table->spec[filter_idx];
 416
 417         efx_filter_init_rx(spec, EFX_FILTER_PRI_MANUAL,
 418                            EFX_FILTER_FLAG_RX_RSS, 0);
 419         spec->type = EFX_FILTER_UC_DEF + filter_idx;
 420         table->used_bitmap[0] |= 1 << filter_idx;
 421 }
 422
 423 int efx_filter_get_eth_local(const struct efx_filter_spec *spec,
 424                              u16 *vid, u8 *addr)
 425 {
 426         switch (spec->type) {
 427         case EFX_FILTER_MAC_WILD:
 428                 *vid = EFX_FILTER_VID_UNSPEC;
 429                 break;
 430         case EFX_FILTER_MAC_FULL:
 431                 *vid = spec->data[0];
 432                 break;
 433         default:
 434                 return -EINVAL;
 435         }
 436
 437         addr[0] = spec->data[2] >> 8;
 438         addr[1] = spec->data[2];
 439         addr[2] = spec->data[1] >> 24;
 440         addr[3] = spec->data[1] >> 16;
 441         addr[4] = spec->data[1] >> 8;
 442         addr[5] = spec->data[1];
 443         return 0;
 444 }
 445
 446 /* Build a filter entry and return its n-tuple key. */
 447 static u32 efx_filter_build(efx_oword_t *filter, struct efx_filter_spec *spec)
 448 {
 449         u32 data3;
 450
 451         switch (efx_filter_spec_table_id(spec)) {
 452         case EFX_FILTER_TABLE_RX_IP: {
 453                 bool is_udp = (spec->type == EFX_FILTER_UDP_FULL ||
 454                                spec->type == EFX_FILTER_UDP_WILD);
 455                 EFX_POPULATE_OWORD_7(
 456                         *filter,
 457                         FRF_BZ_RSS_EN,
 458                         !!(spec->flags & EFX_FILTER_FLAG_RX_RSS),
 459                         FRF_BZ_SCATTER_EN,
 460                         !!(spec->flags & EFX_FILTER_FLAG_RX_SCATTER),
 461                         FRF_BZ_TCP_UDP, is_udp,
 462                         FRF_BZ_RXQ_ID, spec->dmaq_id,
 463                         EFX_DWORD_2, spec->data[2],
 464                         EFX_DWORD_1, spec->data[1],
 465                         EFX_DWORD_0, spec->data[0]);
 466                 data3 = is_udp;
 467                 break;
 468         }
 469
 470         case EFX_FILTER_TABLE_RX_MAC: {
 471                 bool is_wild = spec->type == EFX_FILTER_MAC_WILD;
 472                 EFX_POPULATE_OWORD_7(
 473                         *filter,
 474                         FRF_CZ_RMFT_RSS_EN,
 475                         !!(spec->flags & EFX_FILTER_FLAG_RX_RSS),
 476                         FRF_CZ_RMFT_SCATTER_EN,
 477                         !!(spec->flags & EFX_FILTER_FLAG_RX_SCATTER),
 478                         FRF_CZ_RMFT_RXQ_ID, spec->dmaq_id,
 479                         FRF_CZ_RMFT_WILDCARD_MATCH, is_wild,
 480                         FRF_CZ_RMFT_DEST_MAC_HI, spec->data[2],
 481                         FRF_CZ_RMFT_DEST_MAC_LO, spec->data[1],
 482                         FRF_CZ_RMFT_VLAN_ID, spec->data[0]);
 483                 data3 = is_wild;
 484                 break;
 485         }
 486
 487         case EFX_FILTER_TABLE_TX_MAC: {
 488                 bool is_wild = spec->type == EFX_FILTER_MAC_WILD;
 489                 EFX_POPULATE_OWORD_5(*filter,
 490                                      FRF_CZ_TMFT_TXQ_ID, spec->dmaq_id,
 491                                      FRF_CZ_TMFT_WILDCARD_MATCH, is_wild,
 492                                      FRF_CZ_TMFT_SRC_MAC_HI, spec->data[2],
 493                                      FRF_CZ_TMFT_SRC_MAC_LO, spec->data[1],
 494                                      FRF_CZ_TMFT_VLAN_ID, spec->data[0]);
 495                 data3 = is_wild | spec->dmaq_id << 1;
 496                 break;
 497         }
 498
 499         default:
 500                 BUG();
 501         }
 502
 503         return spec->data[0] ^ spec->data[1] ^ spec->data[2] ^ data3;
 504 }
 505
 506 static bool efx_filter_equal(const struct efx_filter_spec *left,
 507                              const struct efx_filter_spec *right)
 508 {
 509         if (left->type != right->type ||
 510             memcmp(left->data, right->data, sizeof(left->data)))
 511                 return false;
 512
 513         if (left->flags & EFX_FILTER_FLAG_TX &&
 514             left->dmaq_id != right->dmaq_id)
 515                 return false;
 516
 517         return true;
 518 }
 519
 520 /*
 521  * Construct/deconstruct external filter IDs.  At least the RX filter
 522  * IDs must be ordered by matching priority, for RX NFC semantics.
 523  *
 524  * Deconstruction needs to be robust against invalid IDs so that
 525  * efx_filter_remove_id_safe() and efx_filter_get_filter_safe() can
 526  * accept user-provided IDs.
 527  */
 528
 529 #define EFX_FILTER_MATCH_PRI_COUNT      5
 530
 531 static const u8 efx_filter_type_match_pri[EFX_FILTER_TYPE_COUNT] = {
 532         [EFX_FILTER_TCP_FULL]   = 0,
 533         [EFX_FILTER_UDP_FULL]   = 0,
 534         [EFX_FILTER_TCP_WILD]   = 1,
 535         [EFX_FILTER_UDP_WILD]   = 1,
 536         [EFX_FILTER_MAC_FULL]   = 2,
 537         [EFX_FILTER_MAC_WILD]   = 3,
 538         [EFX_FILTER_UC_DEF]     = 4,
 539         [EFX_FILTER_MC_DEF]     = 4,
 540 };
 541
 542 static const enum efx_filter_table_id efx_filter_range_table[] = {
 543         EFX_FILTER_TABLE_RX_IP,         /* RX match pri 0 */
 544         EFX_FILTER_TABLE_RX_IP,
 545         EFX_FILTER_TABLE_RX_MAC,
 546         EFX_FILTER_TABLE_RX_MAC,
 547         EFX_FILTER_TABLE_RX_DEF,        /* RX match pri 4 */
 548         EFX_FILTER_TABLE_COUNT,         /* TX match pri 0; invalid */
 549         EFX_FILTER_TABLE_COUNT,         /* invalid */
 550         EFX_FILTER_TABLE_TX_MAC,
 551         EFX_FILTER_TABLE_TX_MAC,        /* TX match pri 3 */
 552 };
 553
 554 #define EFX_FILTER_INDEX_WIDTH  13
 555 #define EFX_FILTER_INDEX_MASK   ((1 << EFX_FILTER_INDEX_WIDTH) - 1)
 556
 557 static inline u32
 558 efx_filter_make_id(const struct efx_filter_spec *spec, unsigned int index)
 559 {
 560         unsigned int range;
 561
 562         range = efx_filter_type_match_pri[spec->type];
 563         if (!(spec->flags & EFX_FILTER_FLAG_RX))
 564                 range += EFX_FILTER_MATCH_PRI_COUNT;
 565
 566         return range << EFX_FILTER_INDEX_WIDTH | index;
 567 }
 568
 569 static inline enum efx_filter_table_id efx_filter_id_table_id(u32 id)
 570 {
 571         unsigned int range = id >> EFX_FILTER_INDEX_WIDTH;
 572
 573         if (range < ARRAY_SIZE(efx_filter_range_table))
 574                 return efx_filter_range_table[range];
 575         else
 576                 return EFX_FILTER_TABLE_COUNT; /* invalid */
 577 }
 578
 579 static inline unsigned int efx_filter_id_index(u32 id)
 580 {
 581         return id & EFX_FILTER_INDEX_MASK;
 582 }
 583
 584 static inline u8 efx_filter_id_flags(u32 id)
 585 {
 586         unsigned int range = id >> EFX_FILTER_INDEX_WIDTH;
 587
 588         if (range < EFX_FILTER_MATCH_PRI_COUNT)
 589                 return EFX_FILTER_FLAG_RX;
 590         else
 591                 return EFX_FILTER_FLAG_TX;
 592 }
 593
 594 u32 efx_filter_get_rx_id_limit(struct efx_nic *efx)
 595 {
 596         struct efx_filter_state *state = efx->filter_state;
 597         unsigned int range = EFX_FILTER_MATCH_PRI_COUNT - 1;
 598         enum efx_filter_table_id table_id;
 599
 600         do {
 601                 table_id = efx_filter_range_table[range];
 602                 if (state->table[table_id].size != 0)
 603                         return range << EFX_FILTER_INDEX_WIDTH |
 604                                 state->table[table_id].size;
 605         } while (range--);
 606
 607         return 0;
 608 }
 609
 610 /**
 611  * efx_filter_insert_filter - add or replace a filter
 612  * @efx: NIC in which to insert the filter
 613  * @spec: Specification for the filter
 614  * @replace_equal: Flag for whether the specified filter may replace an
 615  *      existing filter with equal priority
 616  *
 617  * On success, return the filter ID.
 618  * On failure, return a negative error code.
 619  *
 620  * If an existing filter has equal match values to the new filter
 621  * spec, then the new filter might replace it, depending on the
 622  * relative priorities.  If the existing filter has lower priority, or
 623  * if @replace_equal is set and it has equal priority, then it is
 624  * replaced.  Otherwise the function fails, returning -%EPERM if
 625  * the existing filter has higher priority or -%EEXIST if it has
 626  * equal priority.
 627  */
 628 s32 efx_filter_insert_filter(struct efx_nic *efx, struct efx_filter_spec *spec,
 629                              bool replace_equal)
 630 {
 631         struct efx_filter_state *state = efx->filter_state;
 632         struct efx_filter_table *table = efx_filter_spec_table(state, spec);
 633         efx_oword_t filter;
 634         int rep_index, ins_index;
 635         unsigned int depth = 0;
 636         int rc;
 637
 638         if (!table || table->size == 0)
 639                 return -EINVAL;
 640
 641         netif_vdbg(efx, hw, efx->net_dev,
 642                    "%s: type %d search_depth=%d", __func__, spec->type,
 643                    table->search_depth[spec->type]);
 644
 645         if (table->id == EFX_FILTER_TABLE_RX_DEF) {
 646                 /* One filter spec per type */
 647                 BUILD_BUG_ON(EFX_FILTER_INDEX_UC_DEF != 0);
 648                 BUILD_BUG_ON(EFX_FILTER_INDEX_MC_DEF !=
 649                              EFX_FILTER_MC_DEF - EFX_FILTER_UC_DEF);
 650                 rep_index = spec->type - EFX_FILTER_INDEX_UC_DEF;
 651                 ins_index = rep_index;
 652
 653                 spin_lock_bh(&state->lock);
 654         } else {
 655                 /* Search concurrently for
 656                  * (1) a filter to be replaced (rep_index): any filter
 657                  *     with the same match values, up to the current
 658                  *     search depth for this type, and
 659                  * (2) the insertion point (ins_index): (1) or any
 660                  *     free slot before it or up to the maximum search
 661                  *     depth for this priority
 662                  * We fail if we cannot find (2).
 663                  *
 664                  * We can stop once either
 665                  * (a) we find (1), in which case we have definitely
 666                  *     found (2) as well; or
 667                  * (b) we have searched exhaustively for (1), and have
 668                  *     either found (2) or searched exhaustively for it
 669                  */
 670                 u32 key = efx_filter_build(&filter, spec);
 671                 unsigned int hash = efx_filter_hash(key);
 672                 unsigned int incr = efx_filter_increment(key);
 673                 unsigned int max_rep_depth = table->search_depth[spec->type];
 674                 unsigned int max_ins_depth =
 675                         spec->priority <= EFX_FILTER_PRI_HINT ?
 676                         FILTER_CTL_SRCH_HINT_MAX : FILTER_CTL_SRCH_MAX;
 677                 unsigned int i = hash & (table->size - 1);
 678
 679                 ins_index = -1;
 680                 depth = 1;
 681
 682                 spin_lock_bh(&state->lock);
 683
 684                 for (;;) {
 685                         if (!test_bit(i, table->used_bitmap)) {
 686                                 if (ins_index < 0)
 687                                         ins_index = i;
 688                         } else if (efx_filter_equal(spec, &table->spec[i])) {
 689                                 /* Case (a) */
 690                                 if (ins_index < 0)
 691                                         ins_index = i;
 692                                 rep_index = i;
 693                                 break;
 694                         }
 695
 696                         if (depth >= max_rep_depth &&
 697                             (ins_index >= 0 || depth >= max_ins_depth)) {
 698                                 /* Case (b) */
 699                                 if (ins_index < 0) {
 700                                         rc = -EBUSY;
 701                                         goto out;
 702                                 }
 703                                 rep_index = -1;
 704                                 break;
 705                         }
 706
 707                         i = (i + incr) & (table->size - 1);
 708                         ++depth;
 709                 }
 710         }
 711
 712         /* If we found a filter to be replaced, check whether we
 713          * should do so
 714          */
 715         if (rep_index >= 0) {
 716                 struct efx_filter_spec *saved_spec = &table->spec[rep_index];
 717
 718                 if (spec->priority == saved_spec->priority && !replace_equal) {
 719                         rc = -EEXIST;
 720                         goto out;
 721                 }
 722                 if (spec->priority < saved_spec->priority) {
 723                         rc = -EPERM;
 724                         goto out;
 725                 }
 726         }
 727
 728         /* Insert the filter */
 729         if (ins_index != rep_index) {
 730                 __set_bit(ins_index, table->used_bitmap);
 731                 ++table->used;
 732         }
 733         table->spec[ins_index] = *spec;
 734
 735         if (table->id == EFX_FILTER_TABLE_RX_DEF) {
 736                 efx_filter_push_rx_config(efx);
 737         } else {
 738                 if (table->search_depth[spec->type] < depth) {
 739                         table->search_depth[spec->type] = depth;
 740                         if (spec->flags & EFX_FILTER_FLAG_TX)
 741                                 efx_filter_push_tx_limits(efx);
 742                         else
 743                                 efx_filter_push_rx_config(efx);
 744                 }
 745
 746                 efx_writeo(efx, &filter,
 747                            table->offset + table->step * ins_index);
 748
 749                 /* If we were able to replace a filter by inserting
 750                  * at a lower depth, clear the replaced filter
 751                  */
 752                 if (ins_index != rep_index && rep_index >= 0)
 753                         efx_filter_table_clear_entry(efx, table, rep_index);
 754         }
 755
 756         netif_vdbg(efx, hw, efx->net_dev,
 757                    "%s: filter type %d index %d rxq %u set",
 758                    __func__, spec->type, ins_index, spec->dmaq_id);
 759         rc = efx_filter_make_id(spec, ins_index);
 760
 761 out:
 762         spin_unlock_bh(&state->lock);
 763         return rc;
 764 }
 765
 766 static void efx_filter_table_clear_entry(struct efx_nic *efx,
 767                                          struct efx_filter_table *table,
 768                                          unsigned int filter_idx)
 769 {
 770         static efx_oword_t filter;
 771
 772         if (table->id == EFX_FILTER_TABLE_RX_DEF) {
 773                 /* RX default filters must always exist */
 774                 efx_filter_reset_rx_def(efx, filter_idx);
 775                 efx_filter_push_rx_config(efx);
 776         } else if (test_bit(filter_idx, table->used_bitmap)) {
 777                 __clear_bit(filter_idx, table->used_bitmap);
 778                 --table->used;
 779                 memset(&table->spec[filter_idx], 0, sizeof(table->spec[0]));
 780
 781                 efx_writeo(efx, &filter,
 782                            table->offset + table->step * filter_idx);
 783         }
 784 }
 785
 786 /**
 787  * efx_filter_remove_id_safe - remove a filter by ID, carefully
 788  * @efx: NIC from which to remove the filter
 789  * @priority: Priority of filter, as passed to @efx_filter_insert_filter
 790  * @filter_id: ID of filter, as returned by @efx_filter_insert_filter
 791  *
 792  * This function will range-check @filter_id, so it is safe to call
 793  * with a value passed from userland.
 794  */
 795 int efx_filter_remove_id_safe(struct efx_nic *efx,
 796                               enum efx_filter_priority priority,
 797                               u32 filter_id)
 798 {
 799         struct efx_filter_state *state = efx->filter_state;
 800         enum efx_filter_table_id table_id;
 801         struct efx_filter_table *table;
 802         unsigned int filter_idx;
 803         struct efx_filter_spec *spec;
 804         u8 filter_flags;
 805         int rc;
 806
 807         table_id = efx_filter_id_table_id(filter_id);
 808         if ((unsigned int)table_id >= EFX_FILTER_TABLE_COUNT)
 809                 return -ENOENT;
 810         table = &state->table[table_id];
 811
 812         filter_idx = efx_filter_id_index(filter_id);
 813         if (filter_idx >= table->size)
 814                 return -ENOENT;
 815         spec = &table->spec[filter_idx];
 816
 817         filter_flags = efx_filter_id_flags(filter_id);
 818
 819         spin_lock_bh(&state->lock);
 820
 821         if (test_bit(filter_idx, table->used_bitmap) &&
 822             spec->priority == priority) {
 823                 efx_filter_table_clear_entry(efx, table, filter_idx);
 824                 if (table->used == 0)
 825                         efx_filter_table_reset_search_depth(table);
 826                 rc = 0;
 827         } else {
 828                 rc = -ENOENT;
 829         }
 830
 831         spin_unlock_bh(&state->lock);
 832
 833         return rc;
 834 }
 835
 836 /**
 837  * efx_filter_get_filter_safe - retrieve a filter by ID, carefully
 838  * @efx: NIC from which to remove the filter
 839  * @priority: Priority of filter, as passed to @efx_filter_insert_filter
 840  * @filter_id: ID of filter, as returned by @efx_filter_insert_filter
 841  * @spec: Buffer in which to store filter specification
 842  *
 843  * This function will range-check @filter_id, so it is safe to call
 844  * with a value passed from userland.
 845  */
 846 int efx_filter_get_filter_safe(struct efx_nic *efx,
 847                                enum efx_filter_priority priority,
 848                                u32 filter_id, struct efx_filter_spec *spec_buf)
 849 {
 850         struct efx_filter_state *state = efx->filter_state;
 851         enum efx_filter_table_id table_id;
 852         struct efx_filter_table *table;
 853         struct efx_filter_spec *spec;
 854         unsigned int filter_idx;
 855         u8 filter_flags;
 856         int rc;
 857
 858         table_id = efx_filter_id_table_id(filter_id);
 859         if ((unsigned int)table_id >= EFX_FILTER_TABLE_COUNT)
 860                 return -ENOENT;
 861         table = &state->table[table_id];
 862
 863         filter_idx = efx_filter_id_index(filter_id);
 864         if (filter_idx >= table->size)
 865                 return -ENOENT;
 866         spec = &table->spec[filter_idx];
 867
 868         filter_flags = efx_filter_id_flags(filter_id);
 869
 870         spin_lock_bh(&state->lock);
 871
 872         if (test_bit(filter_idx, table->used_bitmap) &&
 873             spec->priority == priority) {
 874                 *spec_buf = *spec;
 875                 rc = 0;
 876         } else {
 877                 rc = -ENOENT;
 878         }
 879
 880         spin_unlock_bh(&state->lock);
 881
 882         return rc;
 883 }
 884
 885 static void efx_filter_table_clear(struct efx_nic *efx,
 886                                    enum efx_filter_table_id table_id,
 887                                    enum efx_filter_priority priority)
 888 {
 889         struct efx_filter_state *state = efx->filter_state;
 890         struct efx_filter_table *table = &state->table[table_id];
 891         unsigned int filter_idx;
 892
 893         spin_lock_bh(&state->lock);
 894
 895         for (filter_idx = 0; filter_idx < table->size; ++filter_idx)
 896                 if (table->spec[filter_idx].priority <= priority)
 897                         efx_filter_table_clear_entry(efx, table, filter_idx);
 898         if (table->used == 0)
 899                 efx_filter_table_reset_search_depth(table);
 900
 901         spin_unlock_bh(&state->lock);
 902 }
 903
 904 /**
 905  * efx_filter_clear_rx - remove RX filters by priority
 906  * @efx: NIC from which to remove the filters
 907  * @priority: Maximum priority to remove
 908  */
 909 void efx_filter_clear_rx(struct efx_nic *efx, enum efx_filter_priority priority)
 910 {
 911         efx_filter_table_clear(efx, EFX_FILTER_TABLE_RX_IP, priority);
 912         efx_filter_table_clear(efx, EFX_FILTER_TABLE_RX_MAC, priority);
 913 }
 914
 915 u32 efx_filter_count_rx_used(struct efx_nic *efx,
 916                              enum efx_filter_priority priority)
 917 {
 918         struct efx_filter_state *state = efx->filter_state;
 919         enum efx_filter_table_id table_id;
 920         struct efx_filter_table *table;
 921         unsigned int filter_idx;
 922         u32 count = 0;
 923
 924         spin_lock_bh(&state->lock);
 925
 926         for (table_id = EFX_FILTER_TABLE_RX_IP;
 927              table_id <= EFX_FILTER_TABLE_RX_DEF;
 928              table_id++) {
 929                 table = &state->table[table_id];
 930                 for (filter_idx = 0; filter_idx < table->size; filter_idx++) {
 931                         if (test_bit(filter_idx, table->used_bitmap) &&
 932                             table->spec[filter_idx].priority == priority)
 933                                 ++count;
 934                 }
 935         }
 936
 937         spin_unlock_bh(&state->lock);
 938
 939         return count;
 940 }
 941
 942 s32 efx_filter_get_rx_ids(struct efx_nic *efx,
 943                           enum efx_filter_priority priority,
 944                           u32 *buf, u32 size)
 945 {
 946         struct efx_filter_state *state = efx->filter_state;
 947         enum efx_filter_table_id table_id;
 948         struct efx_filter_table *table;
 949         unsigned int filter_idx;
 950         s32 count = 0;
 951
 952         spin_lock_bh(&state->lock);
 953
 954         for (table_id = EFX_FILTER_TABLE_RX_IP;
 955              table_id <= EFX_FILTER_TABLE_RX_DEF;
 956              table_id++) {
 957                 table = &state->table[table_id];
 958                 for (filter_idx = 0; filter_idx < table->size; filter_idx++) {
 959                         if (test_bit(filter_idx, table->used_bitmap) &&
 960                             table->spec[filter_idx].priority == priority) {
 961                                 if (count == size) {
 962                                         count = -EMSGSIZE;
 963                                         goto out;
 964                                 }
 965                                 buf[count++] = efx_filter_make_id(
 966                                         &table->spec[filter_idx], filter_idx);
 967                         }
 968                 }
 969         }
 970 out:
 971         spin_unlock_bh(&state->lock);
 972
 973         return count;
 974 }
 975
 976 /* Restore filter stater after reset */
 977 void efx_restore_filters(struct efx_nic *efx)
 978 {
 979         struct efx_filter_state *state = efx->filter_state;
 980         enum efx_filter_table_id table_id;
 981         struct efx_filter_table *table;
 982         efx_oword_t filter;
 983         unsigned int filter_idx;
 984
 985         spin_lock_bh(&state->lock);
 986
 987         for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) {
 988                 table = &state->table[table_id];
 989
 990                 /* Check whether this is a regular register table */
 991                 if (table->step == 0)
 992                         continue;
 993
 994                 for (filter_idx = 0; filter_idx < table->size; filter_idx++) {
 995                         if (!test_bit(filter_idx, table->used_bitmap))
 996                                 continue;
 997                         efx_filter_build(&filter, &table->spec[filter_idx]);
 998                         efx_writeo(efx, &filter,
 999                                    table->offset + table->step * filter_idx);
1000                 }
1001         }
1002
1003         efx_filter_push_rx_config(efx);
1004         efx_filter_push_tx_limits(efx);
1005
1006         spin_unlock_bh(&state->lock);
1007 }
1008
1009 int efx_probe_filters(struct efx_nic *efx)
1010 {
1011         struct efx_filter_state *state;
1012         struct efx_filter_table *table;
1013         unsigned table_id;
1014
1015         state = kzalloc(sizeof(*efx->filter_state), GFP_KERNEL);
1016         if (!state)
1017                 return -ENOMEM;
1018         efx->filter_state = state;
1019
1020         spin_lock_init(&state->lock);
1021
1022         if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) {
1023 #ifdef CONFIG_RFS_ACCEL
1024                 state->rps_flow_id = kcalloc(FR_BZ_RX_FILTER_TBL0_ROWS,
1025                                              sizeof(*state->rps_flow_id),
1026                                              GFP_KERNEL);
1027                 if (!state->rps_flow_id)
1028                         goto fail;
1029 #endif
1030                 table = &state->table[EFX_FILTER_TABLE_RX_IP];
1031                 table->id = EFX_FILTER_TABLE_RX_IP;
1032                 table->offset = FR_BZ_RX_FILTER_TBL0;
1033                 table->size = FR_BZ_RX_FILTER_TBL0_ROWS;
1034                 table->step = FR_BZ_RX_FILTER_TBL0_STEP;
1035         }
1036
1037         if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0) {
1038                 table = &state->table[EFX_FILTER_TABLE_RX_MAC];
1039                 table->id = EFX_FILTER_TABLE_RX_MAC;
1040                 table->offset = FR_CZ_RX_MAC_FILTER_TBL0;
1041                 table->size = FR_CZ_RX_MAC_FILTER_TBL0_ROWS;
1042                 table->step = FR_CZ_RX_MAC_FILTER_TBL0_STEP;
1043
1044                 table = &state->table[EFX_FILTER_TABLE_RX_DEF];
1045                 table->id = EFX_FILTER_TABLE_RX_DEF;
1046                 table->size = EFX_FILTER_SIZE_RX_DEF;
1047
1048                 table = &state->table[EFX_FILTER_TABLE_TX_MAC];
1049                 table->id = EFX_FILTER_TABLE_TX_MAC;
1050                 table->offset = FR_CZ_TX_MAC_FILTER_TBL0;
1051                 table->size = FR_CZ_TX_MAC_FILTER_TBL0_ROWS;
1052                 table->step = FR_CZ_TX_MAC_FILTER_TBL0_STEP;
1053         }
1054
1055         for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) {
1056                 table = &state->table[table_id];
1057                 if (table->size == 0)
1058                         continue;
1059                 table->used_bitmap = kcalloc(BITS_TO_LONGS(table->size),
1060                                              sizeof(unsigned long),
1061                                              GFP_KERNEL);
1062                 if (!table->used_bitmap)
1063                         goto fail;
1064                 table->spec = vzalloc(table->size * sizeof(*table->spec));
1065                 if (!table->spec)
1066                         goto fail;
1067         }
1068
1069         if (state->table[EFX_FILTER_TABLE_RX_DEF].size) {
1070                 /* RX default filters must always exist */
1071                 unsigned i;
1072                 for (i = 0; i < EFX_FILTER_SIZE_RX_DEF; i++)
1073                         efx_filter_reset_rx_def(efx, i);
1074         }
1075
1076         efx_filter_push_rx_config(efx);
1077
1078         return 0;
1079
1080 fail:
1081         efx_remove_filters(efx);
1082         return -ENOMEM;
1083 }
1084
1085 void efx_remove_filters(struct efx_nic *efx)
1086 {
1087         struct efx_filter_state *state = efx->filter_state;
1088         enum efx_filter_table_id table_id;
1089
1090         for (table_id = 0; table_id < EFX_FILTER_TABLE_COUNT; table_id++) {
1091                 kfree(state->table[table_id].used_bitmap);
1092                 vfree(state->table[table_id].spec);
1093         }
1094 #ifdef CONFIG_RFS_ACCEL
1095         kfree(state->rps_flow_id);
1096 #endif
1097         kfree(state);
1098 }
1099
1100 #ifdef CONFIG_RFS_ACCEL
1101
1102 int efx_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb,
1103                    u16 rxq_index, u32 flow_id)
1104 {
1105         struct efx_nic *efx = netdev_priv(net_dev);
1106         struct efx_channel *channel;
1107         struct efx_filter_state *state = efx->filter_state;
1108         struct efx_filter_spec spec;
1109         const struct iphdr *ip;
1110         const __be16 *ports;
1111         int nhoff;
1112         int rc;
1113
1114         nhoff = skb_network_offset(skb);
1115
1116         if (skb->protocol != htons(ETH_P_IP))
1117                 return -EPROTONOSUPPORT;
1118
1119         /* RFS must validate the IP header length before calling us */
1120         EFX_BUG_ON_PARANOID(skb_headlen(skb) < nhoff + sizeof(*ip));
1121         ip = (const struct iphdr *)(skb->data + nhoff);
1122         if (ip_is_fragment(ip))
1123                 return -EPROTONOSUPPORT;
1124         EFX_BUG_ON_PARANOID(skb_headlen(skb) < nhoff + 4 * ip->ihl + 4);
1125         ports = (const __be16 *)(skb->data + nhoff + 4 * ip->ihl);
1126
1127         efx_filter_init_rx(&spec, EFX_FILTER_PRI_HINT, 0, rxq_index);
1128         rc = efx_filter_set_ipv4_full(&spec, ip->protocol,
1129                                       ip->daddr, ports[1], ip->saddr, ports[0]);
1130         if (rc)
1131                 return rc;
1132
1133         rc = efx_filter_insert_filter(efx, &spec, true);
1134         if (rc < 0)
1135                 return rc;
1136
1137         /* Remember this so we can check whether to expire the filter later */
1138         state->rps_flow_id[rc] = flow_id;
1139         channel = efx_get_channel(efx, skb_get_rx_queue(skb));
1140         ++channel->rfs_filters_added;
1141
1142         netif_info(efx, rx_status, efx->net_dev,
1143                    "steering %s %pI4:%u:%pI4:%u to queue %u [flow %u filter %d]\n",
1144                    (ip->protocol == IPPROTO_TCP) ? "TCP" : "UDP",
1145                    &ip->saddr, ntohs(ports[0]), &ip->daddr, ntohs(ports[1]),
1146                    rxq_index, flow_id, rc);
1147
1148         return rc;
1149 }
1150
1151 bool __efx_filter_rfs_expire(struct efx_nic *efx, unsigned quota)
1152 {
1153         struct efx_filter_state *state = efx->filter_state;
1154         struct efx_filter_table *table = &state->table[EFX_FILTER_TABLE_RX_IP];
1155         unsigned mask = table->size - 1;
1156         unsigned index;
1157         unsigned stop;
1158
1159         if (!spin_trylock_bh(&state->lock))
1160                 return false;
1161
1162         index = state->rps_expire_index;
1163         stop = (index + quota) & mask;
1164
1165         while (index != stop) {
1166                 if (test_bit(index, table->used_bitmap) &&
1167                     table->spec[index].priority == EFX_FILTER_PRI_HINT &&
1168                     rps_may_expire_flow(efx->net_dev,
1169                                         table->spec[index].dmaq_id,
1170                                         state->rps_flow_id[index], index)) {
1171                         netif_info(efx, rx_status, efx->net_dev,
1172                                    "expiring filter %d [flow %u]\n",
1173                                    index, state->rps_flow_id[index]);
1174                         efx_filter_table_clear_entry(efx, table, index);
1175                 }
1176                 index = (index + 1) & mask;
1177         }
1178
1179         state->rps_expire_index = stop;
1180         if (table->used == 0)
1181                 efx_filter_table_reset_search_depth(table);
1182
1183         spin_unlock_bh(&state->lock);
1184         return true;
1185 }
1186
1187 #endif /* CONFIG_RFS_ACCEL */