1 /*****************************************************************************
5 * $Date: 2005/03/23 07:41:27 $ *
8 * part of the Chelsio 10Gb Ethernet Driver. *
10 * This program is free software; you can redistribute it and/or modify *
11 * it under the terms of the GNU General Public License, version 2, as *
12 * published by the Free Software Foundation. *
14 * You should have received a copy of the GNU General Public License along *
15 * with this program; if not, write to the Free Software Foundation, Inc., *
16 * 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. *
18 * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED *
19 * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF *
20 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. *
22 * http://www.chelsio.com *
24 * Copyright (c) 2003 - 2005 Chelsio Communications, Inc. *
25 * All rights reserved. *
27 * Maintainers: maintainers@chelsio.com *
29 * Authors: Dimitrios Michailidis <dm@chelsio.com> *
30 * Tina Yang <tainay@chelsio.com> *
31 * Felix Marti <felix@chelsio.com> *
32 * Scott Bardone <sbardone@chelsio.com> *
33 * Kurt Ottaway <kottaway@chelsio.com> *
34 * Frank DiMambro <frank@chelsio.com> *
38 ****************************************************************************/
42 #include <linux/config.h>
43 #include <linux/types.h>
44 #include <linux/errno.h>
45 #include <linux/pci.h>
46 #include <linux/netdevice.h>
47 #include <linux/etherdevice.h>
48 #include <linux/if_vlan.h>
49 #include <linux/skbuff.h>
50 #include <linux/init.h>
54 #include <linux/if_arp.h>
61 #include <linux/tcp.h>
64 #define SGE_FREELQ_N 2
65 #define SGE_CMDQ0_E_N 512
66 #define SGE_CMDQ1_E_N 128
67 #define SGE_FREEL_SIZE 4096
68 #define SGE_JUMBO_FREEL_SIZE 512
69 #define SGE_FREEL_REFILL_THRESH 16
70 #define SGE_RESPQ_E_N 1024
71 #define SGE_INTR_BUCKETSIZE 100
72 #define SGE_INTR_LATBUCKETS 5
73 #define SGE_INTR_MAXBUCKETS 11
74 #define SGE_INTRTIMER0 1
75 #define SGE_INTRTIMER1 50
76 #define SGE_INTRTIMER_NRES 10000
77 #define SGE_RX_COPY_THRESHOLD 256
78 #define SGE_RX_SM_BUF_SIZE 1536
80 #define SGE_RESPQ_REPLENISH_THRES ((3 * SGE_RESPQ_E_N) / 4)
82 #define SGE_RX_OFFSET 2
84 # define NET_IP_ALIGN SGE_RX_OFFSET
88 * Memory Mapped HW Command, Freelist and Response Queue Descriptors
90 #if defined(__BIG_ENDIAN_BITFIELD)
93 u32 GenerationBit : 1;
94 u32 BufferLength : 31;
95 u32 RespQueueSelector : 4;
96 u32 ResponseTokens : 12;
103 u32 GenerationBit2 : 1;
109 u32 GenerationBit : 1;
110 u32 BufferLength : 31;
112 u32 GenerationBit2 : 1;
118 u32 Cmdq1CreditReturn : 5;
119 u32 Cmdq1DmaComplete : 5;
120 u32 Cmdq0CreditReturn : 5;
121 u32 Cmdq0DmaComplete : 5;
128 u32 GenerationBit : 1;
132 #elif defined(__LITTLE_ENDIAN_BITFIELD)
134 u32 BufferLength : 31;
135 u32 GenerationBit : 1;
138 u32 GenerationBit2 : 1;
145 u32 ResponseTokens : 12;
146 u32 RespQueueSelector : 4;
150 u32 BufferLength : 31;
151 u32 GenerationBit : 1;
154 u32 GenerationBit2 : 1;
160 u32 GenerationBit : 1;
167 u32 Cmdq0DmaComplete : 5;
168 u32 Cmdq0CreditReturn : 5;
169 u32 Cmdq1DmaComplete : 5;
170 u32 Cmdq1CreditReturn : 5;
176 * SW Context Command and Freelist Queue Descriptors
180 DECLARE_PCI_UNMAP_ADDR(dma_addr);
181 DECLARE_PCI_UNMAP_LEN(dma_len);
187 DECLARE_PCI_UNMAP_ADDR(dma_addr);
188 DECLARE_PCI_UNMAP_LEN(dma_len);
192 * SW Command, Freelist and Response Queue
195 atomic_t asleep; /* HW DMA Fetch status */
196 atomic_t credits; /* # available descriptors for TX */
197 atomic_t pio_pidx; /* Variable updated on Doorbell */
198 u16 entries_n; /* # descriptors for TX */
199 u16 pidx; /* producer index (SW) */
200 u16 cidx; /* consumer index (HW) */
201 u8 genbit; /* current generation (=valid) bit */
202 struct cmdQ_e *entries; /* HW command descriptor Q */
203 struct cmdQ_ce *centries; /* SW command context descriptor Q */
204 spinlock_t Qlock; /* Lock to protect cmdQ enqueuing */
205 dma_addr_t dma_addr; /* DMA addr HW command descriptor Q */
209 unsigned int credits; /* # of available RX buffers */
210 unsigned int entries_n; /* free list capacity */
211 u16 pidx; /* producer index (SW) */
212 u16 cidx; /* consumer index (HW) */
213 u16 rx_buffer_size; /* Buffer size on this free list */
214 u16 dma_offset; /* DMA offset to align IP headers */
215 u8 genbit; /* current generation (=valid) bit */
216 struct freelQ_e *entries; /* HW freelist descriptor Q */
217 struct freelQ_ce *centries; /* SW freelist conext descriptor Q */
218 dma_addr_t dma_addr; /* DMA addr HW freelist descriptor Q */
222 u16 credits; /* # of available respQ descriptors */
223 u16 credits_pend; /* # of not yet returned descriptors */
224 u16 entries_n; /* # of response Q descriptors */
225 u16 pidx; /* producer index (HW) */
226 u16 cidx; /* consumer index (SW) */
227 u8 genbit; /* current generation(=valid) bit */
228 struct respQ_e *entries; /* HW response descriptor Q */
229 dma_addr_t dma_addr; /* DMA addr HW response descriptor Q */
233 * Main SGE data structure
235 * Interrupts are handled by a single CPU and it is likely that on a MP system
236 * the application is migrated to another CPU. In that scenario, we try to
237 * seperate the RX(in irq context) and TX state in order to decrease memory
241 struct adapter *adapter; /* adapter backpointer */
242 struct freelQ freelQ[SGE_FREELQ_N]; /* freelist Q(s) */
243 struct respQ respQ; /* response Q instatiation */
244 unsigned int rx_pkt_pad; /* RX padding for L2 packets */
245 unsigned int jumbo_fl; /* jumbo freelist Q index */
246 u32 intrtimer[SGE_INTR_MAXBUCKETS]; /* ! */
247 u32 currIndex; /* current index into intrtimer[] */
248 u32 intrtimer_nres; /* no resource interrupt timer value */
249 u32 sge_control; /* shadow content of sge control reg */
250 struct sge_intr_counts intr_cnt;
251 struct timer_list ptimer;
252 struct sk_buff *pskb;
254 struct cmdQ cmdQ[SGE_CMDQ_N] ____cacheline_aligned; /* command Q(s)*/
257 static unsigned int t1_sge_tx(struct sk_buff *skb, struct adapter *adapter,
261 * PIO to indicate that memory mapped Q contains valid descriptor(s).
263 static inline void doorbell_pio(struct sge *sge, u32 val)
266 t1_write_reg_4(sge->adapter, A_SG_DOORBELL, val);
270 * Disables the DMA engine.
272 void t1_sge_stop(struct sge *sge)
274 t1_write_reg_4(sge->adapter, A_SG_CONTROL, 0);
275 t1_read_reg_4(sge->adapter, A_SG_CONTROL); /* flush write */
276 if (is_T2(sge->adapter))
277 del_timer_sync(&sge->ptimer);
280 static u8 ch_mac_addr[ETH_ALEN] = {0x0, 0x7, 0x43, 0x0, 0x0, 0x0};
281 static void t1_espi_workaround(void *data)
283 struct adapter *adapter = (struct adapter *)data;
284 struct sge *sge = adapter->sge;
286 if (netif_running(adapter->port[0].dev) &&
287 atomic_read(&sge->cmdQ[0].asleep)) {
289 u32 seop = t1_espi_get_mon(adapter, 0x930, 0);
291 if ((seop & 0xfff0fff) == 0xfff && sge->pskb) {
292 struct sk_buff *skb = sge->pskb;
294 memcpy(skb->data+sizeof(struct cpl_tx_pkt), ch_mac_addr, ETH_ALEN);
295 memcpy(skb->data+skb->len-10, ch_mac_addr, ETH_ALEN);
299 t1_sge_tx(skb, adapter,0);
302 mod_timer(&adapter->sge->ptimer, jiffies + sge->ptimeout);
306 * Enables the DMA engine.
308 void t1_sge_start(struct sge *sge)
310 t1_write_reg_4(sge->adapter, A_SG_CONTROL, sge->sge_control);
311 t1_read_reg_4(sge->adapter, A_SG_CONTROL); /* flush write */
312 if (is_T2(sge->adapter)) {
313 init_timer(&sge->ptimer);
314 sge->ptimer.function = (void *)&t1_espi_workaround;
315 sge->ptimer.data = (unsigned long)sge->adapter;
316 sge->ptimer.expires = jiffies + sge->ptimeout;
317 add_timer(&sge->ptimer);
322 * Creates a t1_sge structure and returns suggested resource parameters.
324 struct sge * __devinit t1_sge_create(struct adapter *adapter,
325 struct sge_params *p)
327 struct sge *sge = kmalloc(sizeof(*sge), GFP_KERNEL);
331 memset(sge, 0, sizeof(*sge));
334 sge->ptimeout = 1; /* finest allowed */
336 sge->adapter = adapter;
337 sge->rx_pkt_pad = t1_is_T1B(adapter) ? 0 : SGE_RX_OFFSET;
338 sge->jumbo_fl = t1_is_T1B(adapter) ? 1 : 0;
340 p->cmdQ_size[0] = SGE_CMDQ0_E_N;
341 p->cmdQ_size[1] = SGE_CMDQ1_E_N;
342 p->freelQ_size[!sge->jumbo_fl] = SGE_FREEL_SIZE;
343 p->freelQ_size[sge->jumbo_fl] = SGE_JUMBO_FREEL_SIZE;
344 p->rx_coalesce_usecs = SGE_INTRTIMER1;
345 p->last_rx_coalesce_raw = SGE_INTRTIMER1 *
346 (board_info(sge->adapter)->clock_core / 1000000);
347 p->default_rx_coalesce_usecs = SGE_INTRTIMER1;
348 p->coalesce_enable = 0; /* Turn off adaptive algorithm by default */
349 p->sample_interval_usecs = 0;
354 * Frees all RX buffers on the freelist Q. The caller must make sure that
355 * the SGE is turned off before calling this function.
357 static void free_freelQ_buffers(struct pci_dev *pdev, struct freelQ *Q)
359 unsigned int cidx = Q->cidx, credits = Q->credits;
362 struct freelQ_ce *ce = &Q->centries[cidx];
364 pci_unmap_single(pdev, pci_unmap_addr(ce, dma_addr),
365 pci_unmap_len(ce, dma_len),
367 dev_kfree_skb(ce->skb);
369 if (++cidx == Q->entries_n)
375 * Free RX free list and response queue resources.
377 static void free_rx_resources(struct sge *sge)
379 struct pci_dev *pdev = sge->adapter->pdev;
380 unsigned int size, i;
382 if (sge->respQ.entries) {
383 size = sizeof(struct respQ_e) * sge->respQ.entries_n;
384 pci_free_consistent(pdev, size, sge->respQ.entries,
385 sge->respQ.dma_addr);
388 for (i = 0; i < SGE_FREELQ_N; i++) {
389 struct freelQ *Q = &sge->freelQ[i];
392 free_freelQ_buffers(pdev, Q);
396 size = sizeof(struct freelQ_e) * Q->entries_n;
397 pci_free_consistent(pdev, size, Q->entries,
404 * Allocates basic RX resources, consisting of memory mapped freelist Qs and a
407 static int alloc_rx_resources(struct sge *sge, struct sge_params *p)
409 struct pci_dev *pdev = sge->adapter->pdev;
410 unsigned int size, i;
412 for (i = 0; i < SGE_FREELQ_N; i++) {
413 struct freelQ *Q = &sge->freelQ[i];
416 Q->entries_n = p->freelQ_size[i];
417 Q->dma_offset = SGE_RX_OFFSET - sge->rx_pkt_pad;
418 size = sizeof(struct freelQ_e) * Q->entries_n;
419 Q->entries = (struct freelQ_e *)
420 pci_alloc_consistent(pdev, size, &Q->dma_addr);
423 memset(Q->entries, 0, size);
424 Q->centries = kcalloc(Q->entries_n, sizeof(struct freelQ_ce),
431 * Calculate the buffer sizes for the two free lists. FL0 accommodates
432 * regular sized Ethernet frames, FL1 is sized not to exceed 16K,
433 * including all the sk_buff overhead.
435 * Note: For T2 FL0 and FL1 are reversed.
437 sge->freelQ[!sge->jumbo_fl].rx_buffer_size = SGE_RX_SM_BUF_SIZE +
438 sizeof(struct cpl_rx_data) +
439 sge->freelQ[!sge->jumbo_fl].dma_offset;
440 sge->freelQ[sge->jumbo_fl].rx_buffer_size = (16 * 1024) -
441 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
443 sge->respQ.genbit = 1;
444 sge->respQ.entries_n = SGE_RESPQ_E_N;
445 sge->respQ.credits = SGE_RESPQ_E_N;
446 size = sizeof(struct respQ_e) * sge->respQ.entries_n;
447 sge->respQ.entries = (struct respQ_e *)
448 pci_alloc_consistent(pdev, size, &sge->respQ.dma_addr);
449 if (!sge->respQ.entries)
451 memset(sge->respQ.entries, 0, size);
455 free_rx_resources(sge);
460 * Frees 'credits_pend' TX buffers and returns the credits to Q->credits.
462 * The adaptive algorithm receives the total size of the buffers freed
463 * accumulated in @*totpayload. No initialization of this argument here.
466 static void free_cmdQ_buffers(struct sge *sge, struct cmdQ *Q,
467 unsigned int credits_pend, unsigned int *totpayload)
469 struct pci_dev *pdev = sge->adapter->pdev;
471 struct cmdQ_ce *ce, *cq = Q->centries;
472 unsigned int entries_n = Q->entries_n, cidx = Q->cidx,
479 pci_unmap_single(pdev, pci_unmap_addr(ce, dma_addr),
480 pci_unmap_len(ce, dma_len),
483 pci_unmap_page(pdev, pci_unmap_addr(ce, dma_addr),
484 pci_unmap_len(ce, dma_len),
487 *totpayload += pci_unmap_len(ce, dma_len);
491 dev_kfree_skb_irq(skb);
494 if (++cidx == entries_n) {
501 atomic_add(credits_pend, &Q->credits);
507 * Assumes that SGE is stopped and all interrupts are disabled.
509 static void free_tx_resources(struct sge *sge)
511 struct pci_dev *pdev = sge->adapter->pdev;
512 unsigned int size, i;
514 for (i = 0; i < SGE_CMDQ_N; i++) {
515 struct cmdQ *Q = &sge->cmdQ[i];
518 unsigned int pending = Q->entries_n -
519 atomic_read(&Q->credits);
522 free_cmdQ_buffers(sge, Q, pending, NULL);
526 size = sizeof(struct cmdQ_e) * Q->entries_n;
527 pci_free_consistent(pdev, size, Q->entries,
534 * Allocates basic TX resources, consisting of memory mapped command Qs.
536 static int alloc_tx_resources(struct sge *sge, struct sge_params *p)
538 struct pci_dev *pdev = sge->adapter->pdev;
539 unsigned int size, i;
541 for (i = 0; i < SGE_CMDQ_N; i++) {
542 struct cmdQ *Q = &sge->cmdQ[i];
545 Q->entries_n = p->cmdQ_size[i];
546 atomic_set(&Q->credits, Q->entries_n);
547 atomic_set(&Q->asleep, 1);
548 spin_lock_init(&Q->Qlock);
549 size = sizeof(struct cmdQ_e) * Q->entries_n;
550 Q->entries = (struct cmdQ_e *)
551 pci_alloc_consistent(pdev, size, &Q->dma_addr);
554 memset(Q->entries, 0, size);
555 Q->centries = kcalloc(Q->entries_n, sizeof(struct cmdQ_ce),
564 free_tx_resources(sge);
568 static inline void setup_ring_params(struct adapter *adapter, u64 addr,
569 u32 size, int base_reg_lo,
570 int base_reg_hi, int size_reg)
572 t1_write_reg_4(adapter, base_reg_lo, (u32)addr);
573 t1_write_reg_4(adapter, base_reg_hi, addr >> 32);
574 t1_write_reg_4(adapter, size_reg, size);
578 * Enable/disable VLAN acceleration.
580 void t1_set_vlan_accel(struct adapter *adapter, int on_off)
582 struct sge *sge = adapter->sge;
584 sge->sge_control &= ~F_VLAN_XTRACT;
586 sge->sge_control |= F_VLAN_XTRACT;
587 if (adapter->open_device_map) {
588 t1_write_reg_4(adapter, A_SG_CONTROL, sge->sge_control);
589 t1_read_reg_4(adapter, A_SG_CONTROL); /* flush */
594 * Sets the interrupt latency timer when the adaptive Rx coalescing
595 * is turned off. Do nothing when it is turned on again.
597 * This routine relies on the fact that the caller has already set
598 * the adaptive policy in adapter->sge_params before calling it.
600 int t1_sge_set_coalesce_params(struct sge *sge, struct sge_params *p)
602 if (!p->coalesce_enable) {
603 u32 newTimer = p->rx_coalesce_usecs *
604 (board_info(sge->adapter)->clock_core / 1000000);
606 t1_write_reg_4(sge->adapter, A_SG_INTRTIMER, newTimer);
612 * Programs the various SGE registers. However, the engine is not yet enabled,
613 * but sge->sge_control is setup and ready to go.
615 static void configure_sge(struct sge *sge, struct sge_params *p)
617 struct adapter *ap = sge->adapter;
620 t1_write_reg_4(ap, A_SG_CONTROL, 0);
621 setup_ring_params(ap, sge->cmdQ[0].dma_addr, sge->cmdQ[0].entries_n,
622 A_SG_CMD0BASELWR, A_SG_CMD0BASEUPR, A_SG_CMD0SIZE);
623 setup_ring_params(ap, sge->cmdQ[1].dma_addr, sge->cmdQ[1].entries_n,
624 A_SG_CMD1BASELWR, A_SG_CMD1BASEUPR, A_SG_CMD1SIZE);
625 setup_ring_params(ap, sge->freelQ[0].dma_addr,
626 sge->freelQ[0].entries_n, A_SG_FL0BASELWR,
627 A_SG_FL0BASEUPR, A_SG_FL0SIZE);
628 setup_ring_params(ap, sge->freelQ[1].dma_addr,
629 sge->freelQ[1].entries_n, A_SG_FL1BASELWR,
630 A_SG_FL1BASEUPR, A_SG_FL1SIZE);
632 /* The threshold comparison uses <. */
633 t1_write_reg_4(ap, A_SG_FLTHRESHOLD, SGE_RX_SM_BUF_SIZE + 1);
635 setup_ring_params(ap, sge->respQ.dma_addr, sge->respQ.entries_n,
636 A_SG_RSPBASELWR, A_SG_RSPBASEUPR, A_SG_RSPSIZE);
637 t1_write_reg_4(ap, A_SG_RSPQUEUECREDIT, (u32)sge->respQ.entries_n);
639 sge->sge_control = F_CMDQ0_ENABLE | F_CMDQ1_ENABLE | F_FL0_ENABLE |
640 F_FL1_ENABLE | F_CPL_ENABLE | F_RESPONSE_QUEUE_ENABLE |
641 V_CMDQ_PRIORITY(2) | F_DISABLE_CMDQ1_GTS | F_ISCSI_COALESCE |
642 V_RX_PKT_OFFSET(sge->rx_pkt_pad);
644 #if defined(__BIG_ENDIAN_BITFIELD)
645 sge->sge_control |= F_ENABLE_BIG_ENDIAN;
649 * Initialize the SGE Interrupt Timer arrray:
650 * intrtimer[0] = (SGE_INTRTIMER0) usec
651 * intrtimer[0<i<5] = (SGE_INTRTIMER0 + i*2) usec
652 * intrtimer[4<i<10] = ((i - 3) * 6) usec
653 * intrtimer[10] = (SGE_INTRTIMER1) usec
656 sge->intrtimer[0] = board_info(sge->adapter)->clock_core / 1000000;
657 for (i = 1; i < SGE_INTR_LATBUCKETS; ++i) {
658 sge->intrtimer[i] = SGE_INTRTIMER0 + (2 * i);
659 sge->intrtimer[i] *= sge->intrtimer[0];
661 for (i = SGE_INTR_LATBUCKETS; i < SGE_INTR_MAXBUCKETS - 1; ++i) {
662 sge->intrtimer[i] = (i - 3) * 6;
663 sge->intrtimer[i] *= sge->intrtimer[0];
665 sge->intrtimer[SGE_INTR_MAXBUCKETS - 1] =
666 sge->intrtimer[0] * SGE_INTRTIMER1;
667 /* Initialize resource timer */
668 sge->intrtimer_nres = sge->intrtimer[0] * SGE_INTRTIMER_NRES;
669 /* Finally finish initialization of intrtimer[0] */
670 sge->intrtimer[0] *= SGE_INTRTIMER0;
671 /* Initialize for a throughput oriented workload */
672 sge->currIndex = SGE_INTR_MAXBUCKETS - 1;
674 if (p->coalesce_enable)
675 t1_write_reg_4(ap, A_SG_INTRTIMER,
676 sge->intrtimer[sge->currIndex]);
678 t1_sge_set_coalesce_params(sge, p);
682 * Return the payload capacity of the jumbo free-list buffers.
684 static inline unsigned int jumbo_payload_capacity(const struct sge *sge)
686 return sge->freelQ[sge->jumbo_fl].rx_buffer_size -
687 sizeof(struct cpl_rx_data) - SGE_RX_OFFSET + sge->rx_pkt_pad;
691 * Allocates both RX and TX resources and configures the SGE. However,
692 * the hardware is not enabled yet.
694 int t1_sge_configure(struct sge *sge, struct sge_params *p)
696 if (alloc_rx_resources(sge, p))
698 if (alloc_tx_resources(sge, p)) {
699 free_rx_resources(sge);
702 configure_sge(sge, p);
705 * Now that we have sized the free lists calculate the payload
706 * capacity of the large buffers. Other parts of the driver use
707 * this to set the max offload coalescing size so that RX packets
708 * do not overflow our large buffers.
710 p->large_buf_capacity = jumbo_payload_capacity(sge);
715 * Frees all SGE related resources and the sge structure itself
717 void t1_sge_destroy(struct sge *sge)
720 dev_kfree_skb(sge->pskb);
721 free_tx_resources(sge);
722 free_rx_resources(sge);
727 * Allocates new RX buffers on the freelist Q (and tracks them on the freelist
728 * context Q) until the Q is full or alloc_skb fails.
730 * It is possible that the generation bits already match, indicating that the
731 * buffer is already valid and nothing needs to be done. This happens when we
732 * copied a received buffer into a new sk_buff during the interrupt processing.
734 * If the SGE doesn't automatically align packets properly (!sge->rx_pkt_pad),
735 * we specify a RX_OFFSET in order to make sure that the IP header is 4B
738 static void refill_free_list(struct sge *sge, struct freelQ *Q)
740 struct pci_dev *pdev = sge->adapter->pdev;
741 struct freelQ_ce *ce = &Q->centries[Q->pidx];
742 struct freelQ_e *e = &Q->entries[Q->pidx];
743 unsigned int dma_len = Q->rx_buffer_size - Q->dma_offset;
746 while (Q->credits < Q->entries_n) {
747 if (e->GenerationBit != Q->genbit) {
751 skb = alloc_skb(Q->rx_buffer_size, GFP_ATOMIC);
755 skb_reserve(skb, Q->dma_offset);
756 mapping = pci_map_single(pdev, skb->data, dma_len,
759 pci_unmap_addr_set(ce, dma_addr, mapping);
760 pci_unmap_len_set(ce, dma_len, dma_len);
761 e->AddrLow = (u32)mapping;
762 e->AddrHigh = (u64)mapping >> 32;
763 e->BufferLength = dma_len;
764 e->GenerationBit = e->GenerationBit2 = Q->genbit;
769 if (++Q->pidx == Q->entries_n) {
781 * Calls refill_free_list for both freelist Qs. If we cannot
782 * fill at least 1/4 of both Qs, we go into 'few interrupt mode' in order
783 * to give the system time to free up resources.
785 static void freelQs_empty(struct sge *sge)
787 u32 irq_reg = t1_read_reg_4(sge->adapter, A_SG_INT_ENABLE);
790 refill_free_list(sge, &sge->freelQ[0]);
791 refill_free_list(sge, &sge->freelQ[1]);
793 if (sge->freelQ[0].credits > (sge->freelQ[0].entries_n >> 2) &&
794 sge->freelQ[1].credits > (sge->freelQ[1].entries_n >> 2)) {
795 irq_reg |= F_FL_EXHAUSTED;
796 irqholdoff_reg = sge->intrtimer[sge->currIndex];
798 /* Clear the F_FL_EXHAUSTED interrupts for now */
799 irq_reg &= ~F_FL_EXHAUSTED;
800 irqholdoff_reg = sge->intrtimer_nres;
802 t1_write_reg_4(sge->adapter, A_SG_INTRTIMER, irqholdoff_reg);
803 t1_write_reg_4(sge->adapter, A_SG_INT_ENABLE, irq_reg);
805 /* We reenable the Qs to force a freelist GTS interrupt later */
806 doorbell_pio(sge, F_FL0_ENABLE | F_FL1_ENABLE);
809 #define SGE_PL_INTR_MASK (F_PL_INTR_SGE_ERR | F_PL_INTR_SGE_DATA)
810 #define SGE_INT_FATAL (F_RESPQ_OVERFLOW | F_PACKET_TOO_BIG | F_PACKET_MISMATCH)
811 #define SGE_INT_ENABLE (F_RESPQ_EXHAUSTED | F_RESPQ_OVERFLOW | \
812 F_FL_EXHAUSTED | F_PACKET_TOO_BIG | F_PACKET_MISMATCH)
815 * Disable SGE Interrupts
817 void t1_sge_intr_disable(struct sge *sge)
819 u32 val = t1_read_reg_4(sge->adapter, A_PL_ENABLE);
821 t1_write_reg_4(sge->adapter, A_PL_ENABLE, val & ~SGE_PL_INTR_MASK);
822 t1_write_reg_4(sge->adapter, A_SG_INT_ENABLE, 0);
826 * Enable SGE interrupts.
828 void t1_sge_intr_enable(struct sge *sge)
830 u32 en = SGE_INT_ENABLE;
831 u32 val = t1_read_reg_4(sge->adapter, A_PL_ENABLE);
833 if (sge->adapter->flags & TSO_CAPABLE)
834 en &= ~F_PACKET_TOO_BIG;
835 t1_write_reg_4(sge->adapter, A_SG_INT_ENABLE, en);
836 t1_write_reg_4(sge->adapter, A_PL_ENABLE, val | SGE_PL_INTR_MASK);
840 * Clear SGE interrupts.
842 void t1_sge_intr_clear(struct sge *sge)
844 t1_write_reg_4(sge->adapter, A_PL_CAUSE, SGE_PL_INTR_MASK);
845 t1_write_reg_4(sge->adapter, A_SG_INT_CAUSE, 0xffffffff);
849 * SGE 'Error' interrupt handler
851 int t1_sge_intr_error_handler(struct sge *sge)
853 struct adapter *adapter = sge->adapter;
854 u32 cause = t1_read_reg_4(adapter, A_SG_INT_CAUSE);
856 if (adapter->flags & TSO_CAPABLE)
857 cause &= ~F_PACKET_TOO_BIG;
858 if (cause & F_RESPQ_EXHAUSTED)
859 sge->intr_cnt.respQ_empty++;
860 if (cause & F_RESPQ_OVERFLOW) {
861 sge->intr_cnt.respQ_overflow++;
862 CH_ALERT("%s: SGE response queue overflow\n",
865 if (cause & F_FL_EXHAUSTED) {
866 sge->intr_cnt.freelistQ_empty++;
869 if (cause & F_PACKET_TOO_BIG) {
870 sge->intr_cnt.pkt_too_big++;
871 CH_ALERT("%s: SGE max packet size exceeded\n",
874 if (cause & F_PACKET_MISMATCH) {
875 sge->intr_cnt.pkt_mismatch++;
876 CH_ALERT("%s: SGE packet mismatch\n", adapter->name);
878 if (cause & SGE_INT_FATAL)
879 t1_fatal_err(adapter);
881 t1_write_reg_4(adapter, A_SG_INT_CAUSE, cause);
886 * The following code is copied from 2.6, where the skb_pull is doing the
887 * right thing and only pulls ETH_HLEN.
889 * Determine the packet's protocol ID. The rule here is that we
890 * assume 802.3 if the type field is short enough to be a length.
891 * This is normal practice and works for any 'now in use' protocol.
893 static unsigned short sge_eth_type_trans(struct sk_buff *skb,
894 struct net_device *dev)
899 skb->mac.raw = skb->data;
900 skb_pull(skb, ETH_HLEN);
901 eth = (struct ethhdr *)skb->mac.raw;
903 if (*eth->h_dest&1) {
904 if(memcmp(eth->h_dest, dev->broadcast, ETH_ALEN) == 0)
905 skb->pkt_type = PACKET_BROADCAST;
907 skb->pkt_type = PACKET_MULTICAST;
911 * This ALLMULTI check should be redundant by 1.4
912 * so don't forget to remove it.
914 * Seems, you forgot to remove it. All silly devices
915 * seems to set IFF_PROMISC.
918 else if (1 /*dev->flags&IFF_PROMISC*/)
920 if(memcmp(eth->h_dest,dev->dev_addr, ETH_ALEN))
921 skb->pkt_type=PACKET_OTHERHOST;
924 if (ntohs(eth->h_proto) >= 1536)
930 * This is a magic hack to spot IPX packets. Older Novell breaks
931 * the protocol design and runs IPX over 802.3 without an 802.2 LLC
932 * layer. We look for FFFF which isn't a used 802.2 SSAP/DSAP. This
933 * won't work for fault tolerant netware but does for the rest.
935 if (*(unsigned short *)rawp == 0xFFFF)
936 return htons(ETH_P_802_3);
941 return htons(ETH_P_802_2);
945 * Prepare the received buffer and pass it up the stack. If it is small enough
946 * and allocation doesn't fail, we use a new sk_buff and copy the content.
948 static unsigned int t1_sge_rx(struct sge *sge, struct freelQ *Q,
949 unsigned int len, unsigned int offload)
952 struct adapter *adapter = sge->adapter;
953 struct freelQ_ce *ce = &Q->centries[Q->cidx];
955 if (len <= SGE_RX_COPY_THRESHOLD &&
956 (skb = alloc_skb(len + NET_IP_ALIGN, GFP_ATOMIC))) {
958 char *src = ce->skb->data;
960 pci_dma_sync_single_for_cpu(adapter->pdev,
961 pci_unmap_addr(ce, dma_addr),
962 pci_unmap_len(ce, dma_len),
965 skb_reserve(skb, NET_IP_ALIGN);
966 src += sge->rx_pkt_pad;
968 memcpy(skb->data, src, len);
970 /* Reuse the entry. */
971 e = &Q->entries[Q->cidx];
972 e->GenerationBit ^= 1;
973 e->GenerationBit2 ^= 1;
975 pci_unmap_single(adapter->pdev, pci_unmap_addr(ce, dma_addr),
976 pci_unmap_len(ce, dma_len),
979 if (!offload && sge->rx_pkt_pad)
980 __skb_pull(skb, sge->rx_pkt_pad);
986 if (unlikely(offload)) {
989 "%s: unexpected offloaded packet, cmd %u\n",
990 adapter->name, *skb->data);
991 dev_kfree_skb_any(skb);
994 struct cpl_rx_pkt *p = (struct cpl_rx_pkt *)skb->data;
996 skb_pull(skb, sizeof(*p));
997 skb->dev = adapter->port[p->iff].dev;
998 skb->dev->last_rx = jiffies;
999 skb->protocol = sge_eth_type_trans(skb, skb->dev);
1000 if ((adapter->flags & RX_CSUM_ENABLED) && p->csum == 0xffff &&
1001 skb->protocol == htons(ETH_P_IP) &&
1002 (skb->data[9] == IPPROTO_TCP ||
1003 skb->data[9] == IPPROTO_UDP))
1004 skb->ip_summed = CHECKSUM_UNNECESSARY;
1006 skb->ip_summed = CHECKSUM_NONE;
1007 if (adapter->vlan_grp && p->vlan_valid)
1008 vlan_hwaccel_rx(skb, adapter->vlan_grp,
1014 if (++Q->cidx == Q->entries_n)
1017 if (unlikely(--Q->credits < Q->entries_n - SGE_FREEL_REFILL_THRESH))
1018 refill_free_list(sge, Q);
1024 * Adaptive interrupt timer logic to keep the CPU utilization to
1025 * manageable levels. Basically, as the Average Packet Size (APS)
1026 * gets higher, the interrupt latency setting gets longer. Every
1027 * SGE_INTR_BUCKETSIZE (of 100B) causes a bump of 2usec to the
1028 * base value of SGE_INTRTIMER0. At large values of payload the
1029 * latency hits the ceiling value of SGE_INTRTIMER1 stored at
1030 * index SGE_INTR_MAXBUCKETS-1 in sge->intrtimer[].
1032 * sge->currIndex caches the last index to save unneeded PIOs.
1034 static inline void update_intr_timer(struct sge *sge, unsigned int avg_payload)
1036 unsigned int newIndex;
1038 newIndex = avg_payload / SGE_INTR_BUCKETSIZE;
1039 if (newIndex > SGE_INTR_MAXBUCKETS - 1) {
1040 newIndex = SGE_INTR_MAXBUCKETS - 1;
1042 /* Save a PIO with this check....maybe */
1043 if (newIndex != sge->currIndex) {
1044 t1_write_reg_4(sge->adapter, A_SG_INTRTIMER,
1045 sge->intrtimer[newIndex]);
1046 sge->currIndex = newIndex;
1047 sge->adapter->params.sge.last_rx_coalesce_raw =
1048 sge->intrtimer[newIndex];
1053 * Returns true if command queue q_num has enough available descriptors that
1054 * we can resume Tx operation after temporarily disabling its packet queue.
1056 static inline int enough_free_Tx_descs(struct sge *sge, int q_num)
1058 return atomic_read(&sge->cmdQ[q_num].credits) >
1059 (sge->cmdQ[q_num].entries_n >> 2);
1063 * Main interrupt handler, optimized assuming that we took a 'DATA'
1066 * 1. Clear the interrupt
1067 * 2. Loop while we find valid descriptors and process them; accumulate
1068 * information that can be processed after the loop
1069 * 3. Tell the SGE at which index we stopped processing descriptors
1070 * 4. Bookkeeping; free TX buffers, ring doorbell if there are any
1071 * outstanding TX buffers waiting, replenish RX buffers, potentially
1072 * reenable upper layers if they were turned off due to lack of TX
1073 * resources which are available again.
1074 * 5. If we took an interrupt, but no valid respQ descriptors was found we
1075 * let the slow_intr_handler run and do error handling.
1077 irqreturn_t t1_interrupt(int irq, void *cookie, struct pt_regs *regs)
1079 struct net_device *netdev;
1080 struct adapter *adapter = cookie;
1081 struct sge *sge = adapter->sge;
1082 struct respQ *Q = &sge->respQ;
1083 unsigned int credits = Q->credits, flags = 0, ret = 0;
1084 unsigned int tot_rxpayload = 0, tot_txpayload = 0, n_rx = 0, n_tx = 0;
1085 unsigned int credits_pend[SGE_CMDQ_N] = { 0, 0 };
1087 struct respQ_e *e = &Q->entries[Q->cidx];
1090 t1_write_reg_4(adapter, A_PL_CAUSE, F_PL_INTR_SGE_DATA);
1093 while (e->GenerationBit == Q->genbit) {
1094 if (--credits < SGE_RESPQ_REPLENISH_THRES) {
1095 u32 n = Q->entries_n - credits - 1;
1097 t1_write_reg_4(adapter, A_SG_RSPQUEUECREDIT, n);
1100 if (likely(e->DataValid)) {
1101 if (!e->Sop || !e->Eop)
1103 t1_sge_rx(sge, &sge->freelQ[e->FreelistQid],
1104 e->BufferLength, e->Offload);
1105 tot_rxpayload += e->BufferLength;
1108 flags |= e->Qsleeping;
1109 credits_pend[0] += e->Cmdq0CreditReturn;
1110 credits_pend[1] += e->Cmdq1CreditReturn;
1114 * If enough cmdQ0 buffers have finished DMAing free them so
1115 * anyone that may be waiting for their release can continue.
1116 * We do this only on MP systems to allow other CPUs to proceed
1117 * promptly. UP systems can wait for the free_cmdQ_buffers()
1118 * calls after this loop as the sole CPU is currently busy in
1121 if (unlikely(credits_pend[0] > SGE_FREEL_REFILL_THRESH)) {
1122 free_cmdQ_buffers(sge, &sge->cmdQ[0], credits_pend[0],
1124 n_tx += credits_pend[0];
1125 credits_pend[0] = 0;
1130 if (unlikely(++Q->cidx == Q->entries_n)) {
1137 Q->credits = credits;
1138 t1_write_reg_4(adapter, A_SG_SLEEPING, Q->cidx);
1140 if (credits_pend[0])
1141 free_cmdQ_buffers(sge, &sge->cmdQ[0], credits_pend[0], &tot_txpayload);
1142 if (credits_pend[1])
1143 free_cmdQ_buffers(sge, &sge->cmdQ[1], credits_pend[1], &tot_txpayload);
1145 /* Do any coalescing and interrupt latency timer adjustments */
1146 if (adapter->params.sge.coalesce_enable) {
1147 unsigned int avg_txpayload = 0, avg_rxpayload = 0;
1149 n_tx += credits_pend[0] + credits_pend[1];
1152 * Choose larger avg. payload size to increase
1153 * throughput and reduce [CPU util., intr/s.]
1155 * Throughput behavior favored in mixed-mode.
1158 avg_txpayload = tot_txpayload/n_tx;
1160 avg_rxpayload = tot_rxpayload/n_rx;
1162 if (n_tx && avg_txpayload > avg_rxpayload){
1163 update_intr_timer(sge, avg_txpayload);
1165 update_intr_timer(sge, avg_rxpayload);
1169 if (flags & F_CMDQ0_ENABLE) {
1170 struct cmdQ *cmdQ = &sge->cmdQ[0];
1172 atomic_set(&cmdQ->asleep, 1);
1173 if (atomic_read(&cmdQ->pio_pidx) != cmdQ->pidx) {
1174 doorbell_pio(sge, F_CMDQ0_ENABLE);
1175 atomic_set(&cmdQ->pio_pidx, cmdQ->pidx);
1178 if (unlikely(flags & (F_FL0_ENABLE | F_FL1_ENABLE)))
1181 netdev = adapter->port[0].dev;
1182 if (unlikely(netif_queue_stopped(netdev) && netif_carrier_ok(netdev) &&
1183 enough_free_Tx_descs(sge, 0) &&
1184 enough_free_Tx_descs(sge, 1))) {
1185 netif_wake_queue(netdev);
1188 ret = t1_slow_intr_handler(adapter);
1190 return IRQ_RETVAL(ret != 0);
1194 * Enqueues the sk_buff onto the cmdQ[qid] and has hardware fetch it.
1196 * The code figures out how many entries the sk_buff will require in the
1197 * cmdQ and updates the cmdQ data structure with the state once the enqueue
1198 * has complete. Then, it doesn't access the global structure anymore, but
1199 * uses the corresponding fields on the stack. In conjuction with a spinlock
1200 * around that code, we can make the function reentrant without holding the
1201 * lock when we actually enqueue (which might be expensive, especially on
1202 * architectures with IO MMUs).
1204 static unsigned int t1_sge_tx(struct sk_buff *skb, struct adapter *adapter,
1207 struct sge *sge = adapter->sge;
1208 struct cmdQ *Q = &sge->cmdQ[qid];
1212 unsigned int credits, pidx, genbit;
1214 unsigned int count = 1 + skb_shinfo(skb)->nr_frags;
1217 * Coming from the timer
1219 if ((skb == sge->pskb)) {
1221 * Quit if any cmdQ activities
1223 if (!spin_trylock(&Q->Qlock))
1225 if (atomic_read(&Q->credits) != Q->entries_n) {
1226 spin_unlock(&Q->Qlock);
1231 spin_lock(&Q->Qlock);
1235 credits = atomic_read(&Q->credits);
1238 atomic_sub(count, &Q->credits);
1240 if (Q->pidx >= Q->entries_n) {
1241 Q->pidx -= Q->entries_n;
1245 if (unlikely(credits < (MAX_SKB_FRAGS + 1))) {
1246 sge->intr_cnt.cmdQ_full[qid]++;
1247 netif_stop_queue(adapter->port[0].dev);
1249 spin_unlock(&Q->Qlock);
1251 mapping = pci_map_single(adapter->pdev, skb->data,
1252 skb->len - skb->data_len, PCI_DMA_TODEVICE);
1253 ce = &Q->centries[pidx];
1255 pci_unmap_addr_set(ce, dma_addr, mapping);
1256 pci_unmap_len_set(ce, dma_len, skb->len - skb->data_len);
1259 e = &Q->entries[pidx];
1262 e->BufferLength = skb->len - skb->data_len;
1263 e->AddrHigh = (u64)mapping >> 32;
1264 e->AddrLow = (u32)mapping;
1271 e->GenerationBit = e->GenerationBit2 = genbit;
1273 for (i = 0; i < count; i++) {
1274 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1277 if (++pidx == Q->entries_n) {
1284 mapping = pci_map_page(adapter->pdev, frag->page,
1289 pci_unmap_addr_set(ce, dma_addr, mapping);
1290 pci_unmap_len_set(ce, dma_len, frag->size);
1295 e->BufferLength = frag->size;
1296 e->AddrHigh = (u64)mapping >> 32;
1297 e->AddrLow = (u32)mapping;
1299 if (i < count - 1) {
1302 e->GenerationBit = e->GenerationBit2 = genbit;
1307 if (skb != sge->pskb)
1311 e->GenerationBit = e->GenerationBit2 = genbit;
1314 * We always ring the doorbell for cmdQ1. For cmdQ0, we only ring
1315 * the doorbell if the Q is asleep. There is a natural race, where
1316 * the hardware is going to sleep just after we checked, however,
1317 * then the interrupt handler will detect the outstanding TX packet
1318 * and ring the doorbell for us.
1321 doorbell_pio(sge, F_CMDQ1_ENABLE);
1322 } else if (atomic_read(&Q->asleep)) {
1323 atomic_set(&Q->asleep, 0);
1324 doorbell_pio(sge, F_CMDQ0_ENABLE);
1325 atomic_set(&Q->pio_pidx, Q->pidx);
1330 #define MK_ETH_TYPE_MSS(type, mss) (((mss) & 0x3FFF) | ((type) << 14))
1333 * Adds the CPL header to the sk_buff and passes it to t1_sge_tx.
1335 int t1_start_xmit(struct sk_buff *skb, struct net_device *dev)
1337 struct adapter *adapter = dev->priv;
1338 struct cpl_tx_pkt *cpl;
1343 * We are using a non-standard hard_header_len and some kernel
1344 * components, such as pktgen, do not handle it right. Complain
1345 * when this happens but try to fix things up.
1347 if (unlikely(skb_headroom(skb) < dev->hard_header_len - ETH_HLEN)) {
1348 struct sk_buff *orig_skb = skb;
1350 if (net_ratelimit())
1352 "%s: Tx packet has inadequate headroom\n",
1354 skb = skb_realloc_headroom(skb, sizeof(struct cpl_tx_pkt_lso));
1355 dev_kfree_skb_any(orig_skb);
1360 if (skb_shinfo(skb)->tso_size) {
1362 struct cpl_tx_pkt_lso *hdr;
1364 eth_type = skb->nh.raw - skb->data == ETH_HLEN ?
1365 CPL_ETH_II : CPL_ETH_II_VLAN;
1367 hdr = (struct cpl_tx_pkt_lso *)skb_push(skb, sizeof(*hdr));
1368 hdr->opcode = CPL_TX_PKT_LSO;
1369 hdr->ip_csum_dis = hdr->l4_csum_dis = 0;
1370 hdr->ip_hdr_words = skb->nh.iph->ihl;
1371 hdr->tcp_hdr_words = skb->h.th->doff;
1372 hdr->eth_type_mss = htons(MK_ETH_TYPE_MSS(eth_type,
1373 skb_shinfo(skb)->tso_size));
1374 hdr->len = htonl(skb->len - sizeof(*hdr));
1375 cpl = (struct cpl_tx_pkt *)hdr;
1379 * An Ethernet packet must have at least space for
1380 * the DIX Ethernet header and be no greater than
1381 * the device set MTU. Otherwise trash the packet.
1383 if (skb->len < ETH_HLEN)
1384 goto t1_start_xmit_fail2;
1385 eth = (struct ethhdr *)skb->data;
1386 if (eth->h_proto == htons(ETH_P_8021Q))
1387 max_len = dev->mtu + VLAN_ETH_HLEN;
1389 max_len = dev->mtu + ETH_HLEN;
1390 if (skb->len > max_len)
1391 goto t1_start_xmit_fail2;
1393 if (!(adapter->flags & UDP_CSUM_CAPABLE) &&
1394 skb->ip_summed == CHECKSUM_HW &&
1395 skb->nh.iph->protocol == IPPROTO_UDP &&
1396 skb_checksum_help(skb, 0))
1397 goto t1_start_xmit_fail3;
1400 if (!adapter->sge->pskb) {
1401 if (skb->protocol == htons(ETH_P_ARP) &&
1402 skb->nh.arph->ar_op == htons(ARPOP_REQUEST))
1403 adapter->sge->pskb = skb;
1405 cpl = (struct cpl_tx_pkt *)skb_push(skb, sizeof(*cpl));
1406 cpl->opcode = CPL_TX_PKT;
1407 cpl->ip_csum_dis = 1; /* SW calculates IP csum */
1408 cpl->l4_csum_dis = skb->ip_summed == CHECKSUM_HW ? 0 : 1;
1409 /* the length field isn't used so don't bother setting it */
1411 cpl->iff = dev->if_port;
1413 #if defined(CONFIG_VLAN_8021Q) || defined(CONFIG_VLAN_8021Q_MODULE)
1414 if (adapter->vlan_grp && vlan_tx_tag_present(skb)) {
1415 cpl->vlan_valid = 1;
1416 cpl->vlan = htons(vlan_tx_tag_get(skb));
1419 cpl->vlan_valid = 0;
1421 dev->trans_start = jiffies;
1422 return t1_sge_tx(skb, adapter, 0);
1424 t1_start_xmit_fail3:
1425 printk(KERN_INFO "%s: Unable to complete checksum\n", dev->name);
1426 goto t1_start_xmit_fail1;
1428 t1_start_xmit_fail2:
1429 printk(KERN_INFO "%s: Invalid packet length %d, dropping\n",
1430 dev->name, skb->len);
1432 t1_start_xmit_fail1:
1433 dev_kfree_skb_any(skb);
1437 void t1_sge_set_ptimeout(adapter_t *adapter, u32 val)
1439 struct sge *sge = adapter->sge;
1442 sge->ptimeout = max((u32)((HZ * val) / 1000), (u32)1);
1445 u32 t1_sge_get_ptimeout(adapter_t *adapter)
1447 struct sge *sge = adapter->sge;
1449 return (is_T2(adapter) ? ((sge->ptimeout * 1000) / HZ) : 0);