2 * Copyright(c) 2004 - 2009 Intel Corporation. All rights reserved.
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License as published by the Free
6 * Software Foundation; either version 2 of the License, or (at your option)
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc., 59
16 * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
18 * The full GNU General Public License is included in this distribution in the
19 * file called COPYING.
24 #include <linux/dmaengine.h>
25 #include <linux/circ_buf.h>
30 extern int ioat_pending_level;
31 extern int ioat_ring_alloc_order;
34 * workaround for IOAT ver.3.0 null descriptor issue
35 * (channel returns error when size is 0)
37 #define NULL_DESC_BUFFER_SIZE 1
39 #define IOAT_MAX_ORDER 16
40 #define ioat_get_alloc_order() \
41 (min(ioat_ring_alloc_order, IOAT_MAX_ORDER))
42 #define ioat_get_max_alloc_order() \
43 (min(ioat_ring_max_alloc_order, IOAT_MAX_ORDER))
45 /* struct ioat2_dma_chan - ioat v2 / v3 channel attributes
46 * @base: common ioat channel parameters
47 * @xfercap_log; log2 of channel max transfer length (for fast division)
48 * @head: allocated index
49 * @issued: hardware notification point
50 * @tail: cleanup index
51 * @dmacount: identical to 'head' except for occasionally resetting to zero
52 * @alloc_order: log2 of the number of allocated descriptors
53 * @produce: number of descriptors to produce at submit time
54 * @ring: software ring buffer implementation of hardware ring
55 * @prep_lock: serializes descriptor preparation (producers)
57 struct ioat2_dma_chan {
58 struct ioat_chan_common base;
66 struct ioat_ring_ent **ring;
70 static inline struct ioat2_dma_chan *to_ioat2_chan(struct dma_chan *c)
72 struct ioat_chan_common *chan = to_chan_common(c);
74 return container_of(chan, struct ioat2_dma_chan, base);
77 static inline u32 ioat2_ring_size(struct ioat2_dma_chan *ioat)
79 return 1 << ioat->alloc_order;
82 /* count of descriptors in flight with the engine */
83 static inline u16 ioat2_ring_active(struct ioat2_dma_chan *ioat)
85 return CIRC_CNT(ioat->head, ioat->tail, ioat2_ring_size(ioat));
88 /* count of descriptors pending submission to hardware */
89 static inline u16 ioat2_ring_pending(struct ioat2_dma_chan *ioat)
91 return CIRC_CNT(ioat->head, ioat->issued, ioat2_ring_size(ioat));
94 static inline u32 ioat2_ring_space(struct ioat2_dma_chan *ioat)
96 return ioat2_ring_size(ioat) - ioat2_ring_active(ioat);
99 static inline u16 ioat2_xferlen_to_descs(struct ioat2_dma_chan *ioat, size_t len)
101 u16 num_descs = len >> ioat->xfercap_log;
103 num_descs += !!(len & ((1 << ioat->xfercap_log) - 1));
108 * struct ioat_ring_ent - wrapper around hardware descriptor
109 * @hw: hardware DMA descriptor (for memcpy)
110 * @fill: hardware fill descriptor
111 * @xor: hardware xor descriptor
112 * @xor_ex: hardware xor extension descriptor
113 * @pq: hardware pq descriptor
114 * @pq_ex: hardware pq extension descriptor
115 * @pqu: hardware pq update descriptor
116 * @raw: hardware raw (un-typed) descriptor
117 * @txd: the generic software descriptor for all engines
118 * @len: total transaction length for unmap
119 * @result: asynchronous result of validate operations
120 * @id: identifier for debug
123 struct ioat_ring_ent {
125 struct ioat_dma_descriptor *hw;
126 struct ioat_fill_descriptor *fill;
127 struct ioat_xor_descriptor *xor;
128 struct ioat_xor_ext_descriptor *xor_ex;
129 struct ioat_pq_descriptor *pq;
130 struct ioat_pq_ext_descriptor *pq_ex;
131 struct ioat_pq_update_descriptor *pqu;
132 struct ioat_raw_descriptor *raw;
135 struct dma_async_tx_descriptor txd;
136 enum sum_check_flags *result;
140 struct ioat_sed_ent *sed;
143 static inline struct ioat_ring_ent *
144 ioat2_get_ring_ent(struct ioat2_dma_chan *ioat, u16 idx)
146 return ioat->ring[idx & (ioat2_ring_size(ioat) - 1)];
149 static inline void ioat2_set_chainaddr(struct ioat2_dma_chan *ioat, u64 addr)
151 struct ioat_chan_common *chan = &ioat->base;
153 writel(addr & 0x00000000FFFFFFFF,
154 chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
156 chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
159 int ioat2_dma_probe(struct ioatdma_device *dev, int dca);
160 int ioat3_dma_probe(struct ioatdma_device *dev, int dca);
161 void ioat3_dma_remove(struct ioatdma_device *dev);
162 struct dca_provider *ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase);
163 struct dca_provider *ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase);
164 int ioat2_check_space_lock(struct ioat2_dma_chan *ioat, int num_descs);
165 int ioat2_enumerate_channels(struct ioatdma_device *device);
166 struct dma_async_tx_descriptor *
167 ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest,
168 dma_addr_t dma_src, size_t len, unsigned long flags);
169 void ioat2_issue_pending(struct dma_chan *chan);
170 int ioat2_alloc_chan_resources(struct dma_chan *c);
171 void ioat2_free_chan_resources(struct dma_chan *c);
172 void __ioat2_restart_chan(struct ioat2_dma_chan *ioat);
173 bool reshape_ring(struct ioat2_dma_chan *ioat, int order);
174 void __ioat2_issue_pending(struct ioat2_dma_chan *ioat);
175 void ioat2_cleanup_event(unsigned long data);
176 void ioat2_timer_event(unsigned long data);
177 int ioat2_quiesce(struct ioat_chan_common *chan, unsigned long tmo);
178 int ioat2_reset_sync(struct ioat_chan_common *chan, unsigned long tmo);
179 extern struct kobj_type ioat2_ktype;
180 extern struct kmem_cache *ioat2_cache;
181 #endif /* IOATDMA_V2_H */