2 * Copyright 2008-2010 Cisco Systems, Inc. All rights reserved.
3 * Copyright 2007 Nuova Systems, Inc. All rights reserved.
5 * This program is free software; you may redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; version 2 of the License.
9 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
10 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
11 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
12 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
13 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
14 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
15 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
23 #include <linux/pci.h>
24 #include <linux/netdevice.h>
29 /* Receive queue control */
31 u64 ring_base; /* 0x00 */
32 u32 ring_size; /* 0x08 */
34 u32 posted_index; /* 0x10 */
36 u32 cq_index; /* 0x18 */
38 u32 enable; /* 0x20 */
40 u32 running; /* 0x28 */
42 u32 fetch_index; /* 0x30 */
44 u32 error_interrupt_enable; /* 0x38 */
46 u32 error_interrupt_offset; /* 0x40 */
48 u32 error_status; /* 0x48 */
50 u32 dropped_packet_count; /* 0x50 */
52 u32 dropped_packet_count_rc; /* 0x58 */
56 /* Break the vnic_rq_buf allocations into blocks of 32/64 entries */
57 #define VNIC_RQ_BUF_MIN_BLK_ENTRIES 32
58 #define VNIC_RQ_BUF_DFLT_BLK_ENTRIES 64
59 #define VNIC_RQ_BUF_BLK_ENTRIES(entries) \
60 ((unsigned int)((entries < VNIC_RQ_BUF_DFLT_BLK_ENTRIES) ? \
61 VNIC_RQ_BUF_MIN_BLK_ENTRIES : VNIC_RQ_BUF_DFLT_BLK_ENTRIES))
62 #define VNIC_RQ_BUF_BLK_SZ(entries) \
63 (VNIC_RQ_BUF_BLK_ENTRIES(entries) * sizeof(struct vnic_rq_buf))
64 #define VNIC_RQ_BUF_BLKS_NEEDED(entries) \
65 DIV_ROUND_UP(entries, VNIC_RQ_BUF_BLK_ENTRIES(entries))
66 #define VNIC_RQ_BUF_BLKS_MAX VNIC_RQ_BUF_BLKS_NEEDED(4096)
69 struct vnic_rq_buf *next;
72 unsigned int os_buf_index;
79 enum enic_poll_state {
87 struct vnic_dev *vdev;
88 struct vnic_rq_ctrl __iomem *ctrl; /* memory-mapped */
89 struct vnic_dev_ring ring;
90 struct vnic_rq_buf *bufs[VNIC_RQ_BUF_BLKS_MAX];
91 struct vnic_rq_buf *to_use;
92 struct vnic_rq_buf *to_clean;
94 unsigned int pkts_outstanding;
95 #ifdef CONFIG_NET_RX_BUSY_POLL
97 #endif /* CONFIG_NET_RX_BUSY_POLL */
100 static inline unsigned int vnic_rq_desc_avail(struct vnic_rq *rq)
102 /* how many does SW own? */
103 return rq->ring.desc_avail;
106 static inline unsigned int vnic_rq_desc_used(struct vnic_rq *rq)
108 /* how many does HW own? */
109 return rq->ring.desc_count - rq->ring.desc_avail - 1;
112 static inline void *vnic_rq_next_desc(struct vnic_rq *rq)
114 return rq->to_use->desc;
117 static inline unsigned int vnic_rq_next_index(struct vnic_rq *rq)
119 return rq->to_use->index;
122 static inline void vnic_rq_post(struct vnic_rq *rq,
123 void *os_buf, unsigned int os_buf_index,
124 dma_addr_t dma_addr, unsigned int len,
127 struct vnic_rq_buf *buf = rq->to_use;
129 buf->os_buf = os_buf;
130 buf->os_buf_index = os_buf_index;
131 buf->dma_addr = dma_addr;
137 rq->ring.desc_avail--;
139 /* Move the posted_index every nth descriptor
142 #ifndef VNIC_RQ_RETURN_RATE
143 #define VNIC_RQ_RETURN_RATE 0xf /* keep 2^n - 1 */
146 if ((buf->index & VNIC_RQ_RETURN_RATE) == 0) {
147 /* Adding write memory barrier prevents compiler and/or CPU
148 * reordering, thus avoiding descriptor posting before
149 * descriptor is initialized. Otherwise, hardware can read
150 * stale descriptor fields.
153 iowrite32(buf->index, &rq->ctrl->posted_index);
157 static inline void vnic_rq_return_descs(struct vnic_rq *rq, unsigned int count)
159 rq->ring.desc_avail += count;
162 enum desc_return_options {
164 VNIC_RQ_DEFER_RETURN_DESC,
167 static inline void vnic_rq_service(struct vnic_rq *rq,
168 struct cq_desc *cq_desc, u16 completed_index,
169 int desc_return, void (*buf_service)(struct vnic_rq *rq,
170 struct cq_desc *cq_desc, struct vnic_rq_buf *buf,
171 int skipped, void *opaque), void *opaque)
173 struct vnic_rq_buf *buf;
179 skipped = (buf->index != completed_index);
181 (*buf_service)(rq, cq_desc, buf, skipped, opaque);
183 if (desc_return == VNIC_RQ_RETURN_DESC)
184 rq->ring.desc_avail++;
186 rq->to_clean = buf->next;
195 static inline int vnic_rq_fill(struct vnic_rq *rq,
196 int (*buf_fill)(struct vnic_rq *rq))
200 while (vnic_rq_desc_avail(rq) > 0) {
202 err = (*buf_fill)(rq);
210 #ifdef CONFIG_NET_RX_BUSY_POLL
211 static inline void enic_busy_poll_init_lock(struct vnic_rq *rq)
213 atomic_set(&rq->bpoll_state, ENIC_POLL_STATE_IDLE);
216 static inline bool enic_poll_lock_napi(struct vnic_rq *rq)
218 int rc = atomic_cmpxchg(&rq->bpoll_state, ENIC_POLL_STATE_IDLE,
219 ENIC_POLL_STATE_NAPI);
221 return (rc == ENIC_POLL_STATE_IDLE);
224 static inline void enic_poll_unlock_napi(struct vnic_rq *rq,
225 struct napi_struct *napi)
227 WARN_ON(atomic_read(&rq->bpoll_state) != ENIC_POLL_STATE_NAPI);
228 napi_gro_flush(napi, false);
229 atomic_set(&rq->bpoll_state, ENIC_POLL_STATE_IDLE);
232 static inline bool enic_poll_lock_poll(struct vnic_rq *rq)
234 int rc = atomic_cmpxchg(&rq->bpoll_state, ENIC_POLL_STATE_IDLE,
235 ENIC_POLL_STATE_POLL);
237 return (rc == ENIC_POLL_STATE_IDLE);
241 static inline void enic_poll_unlock_poll(struct vnic_rq *rq)
243 WARN_ON(atomic_read(&rq->bpoll_state) != ENIC_POLL_STATE_POLL);
244 atomic_set(&rq->bpoll_state, ENIC_POLL_STATE_IDLE);
247 static inline bool enic_poll_busy_polling(struct vnic_rq *rq)
249 return atomic_read(&rq->bpoll_state) & ENIC_POLL_STATE_POLL;
254 static inline void enic_busy_poll_init_lock(struct vnic_rq *rq)
258 static inline bool enic_poll_lock_napi(struct vnic_rq *rq)
263 static inline bool enic_poll_unlock_napi(struct vnic_rq *rq,
264 struct napi_struct *napi)
269 static inline bool enic_poll_lock_poll(struct vnic_rq *rq)
274 static inline bool enic_poll_unlock_poll(struct vnic_rq *rq)
279 static inline bool enic_poll_ll_polling(struct vnic_rq *rq)
283 #endif /* CONFIG_NET_RX_BUSY_POLL */
285 void vnic_rq_free(struct vnic_rq *rq);
286 int vnic_rq_alloc(struct vnic_dev *vdev, struct vnic_rq *rq, unsigned int index,
287 unsigned int desc_count, unsigned int desc_size);
288 void vnic_rq_init(struct vnic_rq *rq, unsigned int cq_index,
289 unsigned int error_interrupt_enable,
290 unsigned int error_interrupt_offset);
291 unsigned int vnic_rq_error_status(struct vnic_rq *rq);
292 void vnic_rq_enable(struct vnic_rq *rq);
293 int vnic_rq_disable(struct vnic_rq *rq);
294 void vnic_rq_clean(struct vnic_rq *rq,
295 void (*buf_clean)(struct vnic_rq *rq, struct vnic_rq_buf *buf));
297 #endif /* _VNIC_RQ_H_ */