2 * bfin_dma.c - Blackfin DMA implementation
4 * Copyright 2004-2008 Analog Devices Inc.
6 * Licensed under the GPL-2 or later.
9 #include <linux/errno.h>
10 #include <linux/interrupt.h>
11 #include <linux/kernel.h>
12 #include <linux/module.h>
13 #include <linux/param.h>
14 #include <linux/proc_fs.h>
15 #include <linux/sched.h>
16 #include <linux/seq_file.h>
17 #include <linux/spinlock.h>
19 #include <asm/blackfin.h>
20 #include <asm/cacheflush.h>
22 #include <asm/uaccess.h>
23 #include <asm/early_printk.h>
26 * To make sure we work around 05000119 - we always check DMA_DONE bit,
27 * never the DMA_RUN bit
30 struct dma_channel dma_ch[MAX_DMA_CHANNELS];
31 EXPORT_SYMBOL(dma_ch);
33 static int __init blackfin_dma_init(void)
37 printk(KERN_INFO "Blackfin DMA Controller\n");
41 bfin_write_DMAC_TC_PER(0x0111);
44 for (i = 0; i < MAX_DMA_CHANNELS; i++) {
45 atomic_set(&dma_ch[i].chan_status, 0);
46 dma_ch[i].regs = dma_io_base_addr[i];
48 #ifdef CH_MEM_STREAM3_SRC
49 /* Mark MEMDMA Channel 3 as requested since we're using it internally */
50 request_dma(CH_MEM_STREAM3_DEST, "Blackfin dma_memcpy");
51 request_dma(CH_MEM_STREAM3_SRC, "Blackfin dma_memcpy");
53 /* Mark MEMDMA Channel 0 as requested since we're using it internally */
54 request_dma(CH_MEM_STREAM0_DEST, "Blackfin dma_memcpy");
55 request_dma(CH_MEM_STREAM0_SRC, "Blackfin dma_memcpy");
58 #if defined(CONFIG_DEB_DMA_URGENT)
59 bfin_write_EBIU_DDRQUE(bfin_read_EBIU_DDRQUE()
60 | DEB1_URGENT | DEB2_URGENT | DEB3_URGENT);
65 arch_initcall(blackfin_dma_init);
68 static int proc_dma_show(struct seq_file *m, void *v)
72 for (i = 0; i < MAX_DMA_CHANNELS; ++i)
73 if (dma_channel_active(i))
74 seq_printf(m, "%2d: %s\n", i, dma_ch[i].device_id);
79 static int proc_dma_open(struct inode *inode, struct file *file)
81 return single_open(file, proc_dma_show, NULL);
84 static const struct file_operations proc_dma_operations = {
85 .open = proc_dma_open,
88 .release = single_release,
91 static int __init proc_dma_init(void)
93 return proc_create("dma", 0, NULL, &proc_dma_operations) != NULL;
95 late_initcall(proc_dma_init);
98 static void set_dma_peripheral_map(unsigned int channel, const char *device_id)
101 unsigned int per_map;
104 case CH_UART2_RX: per_map = 0xC << 12; break;
105 case CH_UART2_TX: per_map = 0xD << 12; break;
106 case CH_UART3_RX: per_map = 0xE << 12; break;
107 case CH_UART3_TX: per_map = 0xF << 12; break;
111 if (strncmp(device_id, "BFIN_UART", 9) == 0)
112 dma_ch[channel].regs->peripheral_map = per_map;
117 * request_dma - request a DMA channel
119 * Request the specific DMA channel from the system if it's available.
121 int request_dma(unsigned int channel, const char *device_id)
123 pr_debug("request_dma() : BEGIN\n");
125 if (device_id == NULL)
126 printk(KERN_WARNING "request_dma(%u): no device_id given\n", channel);
128 #if defined(CONFIG_BF561) && ANOMALY_05000182
129 if (channel >= CH_IMEM_STREAM0_DEST && channel <= CH_IMEM_STREAM1_DEST) {
130 if (get_cclk() > 500000000) {
132 "Request IMDMA failed due to ANOMALY 05000182\n");
138 if (atomic_cmpxchg(&dma_ch[channel].chan_status, 0, 1)) {
139 pr_debug("DMA CHANNEL IN USE\n");
143 set_dma_peripheral_map(channel, device_id);
144 dma_ch[channel].device_id = device_id;
145 dma_ch[channel].irq = 0;
147 /* This is to be enabled by putting a restriction -
148 * you have to request DMA, before doing any operations on
151 pr_debug("request_dma() : END\n");
154 EXPORT_SYMBOL(request_dma);
156 int set_dma_callback(unsigned int channel, irq_handler_t callback, void *data)
161 BUG_ON(channel >= MAX_DMA_CHANNELS || !callback ||
162 !atomic_read(&dma_ch[channel].chan_status));
164 irq = channel2irq(channel);
165 ret = request_irq(irq, callback, 0, dma_ch[channel].device_id, data);
169 dma_ch[channel].irq = irq;
170 dma_ch[channel].data = data;
174 EXPORT_SYMBOL(set_dma_callback);
177 * clear_dma_buffer - clear DMA fifos for specified channel
179 * Set the Buffer Clear bit in the Configuration register of specific DMA
180 * channel. This will stop the descriptor based DMA operation.
182 static void clear_dma_buffer(unsigned int channel)
184 dma_ch[channel].regs->cfg |= RESTART;
186 dma_ch[channel].regs->cfg &= ~RESTART;
189 void free_dma(unsigned int channel)
191 pr_debug("freedma() : BEGIN\n");
192 BUG_ON(channel >= MAX_DMA_CHANNELS ||
193 !atomic_read(&dma_ch[channel].chan_status));
196 disable_dma(channel);
197 clear_dma_buffer(channel);
199 if (dma_ch[channel].irq)
200 free_irq(dma_ch[channel].irq, dma_ch[channel].data);
202 /* Clear the DMA Variable in the Channel */
203 atomic_set(&dma_ch[channel].chan_status, 0);
205 pr_debug("freedma() : END\n");
207 EXPORT_SYMBOL(free_dma);
210 # ifndef MAX_DMA_SUSPEND_CHANNELS
211 # define MAX_DMA_SUSPEND_CHANNELS MAX_DMA_CHANNELS
213 # ifndef CONFIG_BF60x
214 int blackfin_dma_suspend(void)
218 for (i = 0; i < MAX_DMA_CHANNELS; ++i) {
219 if (dma_ch[i].regs->cfg & DMAEN) {
220 printk(KERN_ERR "DMA Channel %d failed to suspend\n", i);
223 if (i < MAX_DMA_SUSPEND_CHANNELS)
224 dma_ch[i].saved_peripheral_map = dma_ch[i].regs->peripheral_map;
228 bfin_write_DMAC_TC_PER(0x0);
233 void blackfin_dma_resume(void)
237 for (i = 0; i < MAX_DMA_CHANNELS; ++i) {
238 dma_ch[i].regs->cfg = 0;
239 if (i < MAX_DMA_SUSPEND_CHANNELS)
240 dma_ch[i].regs->peripheral_map = dma_ch[i].saved_peripheral_map;
243 bfin_write_DMAC_TC_PER(0x0111);
247 int blackfin_dma_suspend(void)
252 void blackfin_dma_resume(void)
259 * blackfin_dma_early_init - minimal DMA init
261 * Setup a few DMA registers so we can safely do DMA transfers early on in
262 * the kernel booting process. Really this just means using dma_memcpy().
264 void __init blackfin_dma_early_init(void)
266 early_shadow_stamp();
267 bfin_write_MDMA_S0_CONFIG(0);
268 bfin_write_MDMA_S1_CONFIG(0);
271 void __init early_dma_memcpy(void *pdst, const void *psrc, size_t size)
273 unsigned long dst = (unsigned long)pdst;
274 unsigned long src = (unsigned long)psrc;
275 struct dma_register *dst_ch, *src_ch;
277 early_shadow_stamp();
279 /* We assume that everything is 4 byte aligned, so include
280 * a basic sanity check
287 /* Find an avalible memDMA channel */
289 if (src_ch == (struct dma_register *)MDMA_S0_NEXT_DESC_PTR) {
290 dst_ch = (struct dma_register *)MDMA_D1_NEXT_DESC_PTR;
291 src_ch = (struct dma_register *)MDMA_S1_NEXT_DESC_PTR;
293 dst_ch = (struct dma_register *)MDMA_D0_NEXT_DESC_PTR;
294 src_ch = (struct dma_register *)MDMA_S0_NEXT_DESC_PTR;
297 if (!DMA_MMR_READ(&src_ch->cfg))
299 else if (DMA_MMR_READ(&dst_ch->irq_status) & DMA_DONE) {
300 DMA_MMR_WRITE(&src_ch->cfg, 0);
305 /* Force a sync in case a previous config reset on this channel
306 * occurred. This is needed so subsequent writes to DMA registers
307 * are not spuriously lost/corrupted.
309 __builtin_bfin_ssync();
312 bfin_write32(&dst_ch->start_addr, dst);
313 DMA_MMR_WRITE(&dst_ch->x_count, size >> 2);
314 DMA_MMR_WRITE(&dst_ch->x_modify, 1 << 2);
315 DMA_MMR_WRITE(&dst_ch->irq_status, DMA_DONE | DMA_ERR);
318 bfin_write32(&src_ch->start_addr, src);
319 DMA_MMR_WRITE(&src_ch->x_count, size >> 2);
320 DMA_MMR_WRITE(&src_ch->x_modify, 1 << 2);
321 DMA_MMR_WRITE(&src_ch->irq_status, DMA_DONE | DMA_ERR);
324 DMA_MMR_WRITE(&src_ch->cfg, DMAEN | WDSIZE_32);
325 DMA_MMR_WRITE(&dst_ch->cfg, WNR | DI_EN_X | DMAEN | WDSIZE_32);
327 /* Since we are atomic now, don't use the workaround ssync */
328 __builtin_bfin_ssync();
331 /* Work around a possible MDMA anomaly. Running 2 MDMA channels to
332 * transfer DDR data to L1 SRAM may corrupt data.
333 * Should be reverted after this issue is root caused.
335 while (!(DMA_MMR_READ(&dst_ch->irq_status) & DMA_DONE))
340 void __init early_dma_memcpy_done(void)
342 early_shadow_stamp();
344 while ((bfin_read_MDMA_S0_CONFIG() && !(bfin_read_MDMA_D0_IRQ_STATUS() & DMA_DONE)) ||
345 (bfin_read_MDMA_S1_CONFIG() && !(bfin_read_MDMA_D1_IRQ_STATUS() & DMA_DONE)))
348 bfin_write_MDMA_D0_IRQ_STATUS(DMA_DONE | DMA_ERR);
349 bfin_write_MDMA_D1_IRQ_STATUS(DMA_DONE | DMA_ERR);
351 * Now that DMA is done, we would normally flush cache, but
352 * i/d cache isn't running this early, so we don't bother,
353 * and just clear out the DMA channel for next time
355 bfin_write_MDMA_S0_CONFIG(0);
356 bfin_write_MDMA_S1_CONFIG(0);
357 bfin_write_MDMA_D0_CONFIG(0);
358 bfin_write_MDMA_D1_CONFIG(0);
360 __builtin_bfin_ssync();
363 #ifdef CH_MEM_STREAM3_SRC
364 #define bfin_read_MDMA_S_CONFIG bfin_read_MDMA_S3_CONFIG
365 #define bfin_write_MDMA_S_CONFIG bfin_write_MDMA_S3_CONFIG
366 #define bfin_write_MDMA_S_START_ADDR bfin_write_MDMA_S3_START_ADDR
367 #define bfin_write_MDMA_S_IRQ_STATUS bfin_write_MDMA_S3_IRQ_STATUS
368 #define bfin_write_MDMA_S_X_COUNT bfin_write_MDMA_S3_X_COUNT
369 #define bfin_write_MDMA_S_X_MODIFY bfin_write_MDMA_S3_X_MODIFY
370 #define bfin_write_MDMA_S_Y_COUNT bfin_write_MDMA_S3_Y_COUNT
371 #define bfin_write_MDMA_S_Y_MODIFY bfin_write_MDMA_S3_Y_MODIFY
372 #define bfin_write_MDMA_D_CONFIG bfin_write_MDMA_D3_CONFIG
373 #define bfin_write_MDMA_D_START_ADDR bfin_write_MDMA_D3_START_ADDR
374 #define bfin_read_MDMA_D_IRQ_STATUS bfin_read_MDMA_D3_IRQ_STATUS
375 #define bfin_write_MDMA_D_IRQ_STATUS bfin_write_MDMA_D3_IRQ_STATUS
376 #define bfin_write_MDMA_D_X_COUNT bfin_write_MDMA_D3_X_COUNT
377 #define bfin_write_MDMA_D_X_MODIFY bfin_write_MDMA_D3_X_MODIFY
378 #define bfin_write_MDMA_D_Y_COUNT bfin_write_MDMA_D3_Y_COUNT
379 #define bfin_write_MDMA_D_Y_MODIFY bfin_write_MDMA_D3_Y_MODIFY
381 #define bfin_read_MDMA_S_CONFIG bfin_read_MDMA_S0_CONFIG
382 #define bfin_write_MDMA_S_CONFIG bfin_write_MDMA_S0_CONFIG
383 #define bfin_write_MDMA_S_START_ADDR bfin_write_MDMA_S0_START_ADDR
384 #define bfin_write_MDMA_S_IRQ_STATUS bfin_write_MDMA_S0_IRQ_STATUS
385 #define bfin_write_MDMA_S_X_COUNT bfin_write_MDMA_S0_X_COUNT
386 #define bfin_write_MDMA_S_X_MODIFY bfin_write_MDMA_S0_X_MODIFY
387 #define bfin_write_MDMA_S_Y_COUNT bfin_write_MDMA_S0_Y_COUNT
388 #define bfin_write_MDMA_S_Y_MODIFY bfin_write_MDMA_S0_Y_MODIFY
389 #define bfin_write_MDMA_D_CONFIG bfin_write_MDMA_D0_CONFIG
390 #define bfin_write_MDMA_D_START_ADDR bfin_write_MDMA_D0_START_ADDR
391 #define bfin_read_MDMA_D_IRQ_STATUS bfin_read_MDMA_D0_IRQ_STATUS
392 #define bfin_write_MDMA_D_IRQ_STATUS bfin_write_MDMA_D0_IRQ_STATUS
393 #define bfin_write_MDMA_D_X_COUNT bfin_write_MDMA_D0_X_COUNT
394 #define bfin_write_MDMA_D_X_MODIFY bfin_write_MDMA_D0_X_MODIFY
395 #define bfin_write_MDMA_D_Y_COUNT bfin_write_MDMA_D0_Y_COUNT
396 #define bfin_write_MDMA_D_Y_MODIFY bfin_write_MDMA_D0_Y_MODIFY
400 * __dma_memcpy - program the MDMA registers
402 * Actually program MDMA0 and wait for the transfer to finish. Disable IRQs
403 * while programming registers so that everything is fully configured. Wait
404 * for DMA to finish with IRQs enabled. If interrupted, the initial DMA_DONE
405 * check will make sure we don't clobber any existing transfer.
407 static void __dma_memcpy(u32 daddr, s16 dmod, u32 saddr, s16 smod, size_t cnt, u32 conf)
409 static DEFINE_SPINLOCK(mdma_lock);
412 spin_lock_irqsave(&mdma_lock, flags);
414 /* Force a sync in case a previous config reset on this channel
415 * occurred. This is needed so subsequent writes to DMA registers
416 * are not spuriously lost/corrupted. Do it under irq lock and
417 * without the anomaly version (because we are atomic already).
419 __builtin_bfin_ssync();
421 if (bfin_read_MDMA_S_CONFIG())
422 while (!(bfin_read_MDMA_D_IRQ_STATUS() & DMA_DONE))
426 /* For larger bit sizes, we've already divided down cnt so it
427 * is no longer a multiple of 64k. So we have to break down
428 * the limit here so it is a multiple of the incoming size.
429 * There is no limitation here in terms of total size other
430 * than the hardware though as the bits lost in the shift are
431 * made up by MODIFY (== we can hit the whole address space).
432 * X: (2^(16 - 0)) * 1 == (2^(16 - 1)) * 2 == (2^(16 - 2)) * 4
434 u32 shift = abs(dmod) >> 1;
435 size_t ycnt = cnt >> (16 - shift);
436 cnt = 1 << (16 - shift);
437 bfin_write_MDMA_D_Y_COUNT(ycnt);
438 bfin_write_MDMA_S_Y_COUNT(ycnt);
439 bfin_write_MDMA_D_Y_MODIFY(dmod);
440 bfin_write_MDMA_S_Y_MODIFY(smod);
443 bfin_write_MDMA_D_START_ADDR(daddr);
444 bfin_write_MDMA_D_X_COUNT(cnt);
445 bfin_write_MDMA_D_X_MODIFY(dmod);
446 bfin_write_MDMA_D_IRQ_STATUS(DMA_DONE | DMA_ERR);
448 bfin_write_MDMA_S_START_ADDR(saddr);
449 bfin_write_MDMA_S_X_COUNT(cnt);
450 bfin_write_MDMA_S_X_MODIFY(smod);
451 bfin_write_MDMA_S_IRQ_STATUS(DMA_DONE | DMA_ERR);
453 bfin_write_MDMA_S_CONFIG(DMAEN | conf);
455 bfin_write_MDMA_D_CONFIG(WNR | DI_EN_Y | DMAEN | conf);
457 bfin_write_MDMA_D_CONFIG(WNR | DI_EN_X | DMAEN | conf);
459 spin_unlock_irqrestore(&mdma_lock, flags);
463 while (!(bfin_read_MDMA_D_IRQ_STATUS() & DMA_DONE))
464 if (bfin_read_MDMA_S_CONFIG())
469 bfin_write_MDMA_D_IRQ_STATUS(DMA_DONE | DMA_ERR);
471 bfin_write_MDMA_S_CONFIG(0);
472 bfin_write_MDMA_D_CONFIG(0);
476 * _dma_memcpy - translate C memcpy settings into MDMA settings
478 * Handle all the high level steps before we touch the MDMA registers. So
479 * handle direction, tweaking of sizes, and formatting of addresses.
481 static void *_dma_memcpy(void *pdst, const void *psrc, size_t size)
485 unsigned long dst = (unsigned long)pdst;
486 unsigned long src = (unsigned long)psrc;
491 if (dst % 4 == 0 && src % 4 == 0 && size % 4 == 0) {
494 } else if (dst % 2 == 0 && src % 2 == 0 && size % 2 == 0) {
502 /* If the two memory regions have a chance of overlapping, make
503 * sure the memcpy still works as expected. Do this by having the
504 * copy run backwards instead.
514 #ifndef DMA_MMR_SIZE_32
519 __dma_memcpy(dst, mod, src, mod, size, conf);
525 * dma_memcpy - DMA memcpy under mutex lock
527 * Do not check arguments before starting the DMA memcpy. Break the transfer
528 * up into two pieces. The first transfer is in multiples of 64k and the
529 * second transfer is the piece smaller than 64k.
531 void *dma_memcpy(void *pdst, const void *psrc, size_t size)
533 unsigned long dst = (unsigned long)pdst;
534 unsigned long src = (unsigned long)psrc;
536 if (bfin_addr_dcacheable(src))
537 blackfin_dcache_flush_range(src, src + size);
539 if (bfin_addr_dcacheable(dst))
540 blackfin_dcache_invalidate_range(dst, dst + size);
542 return dma_memcpy_nocache(pdst, psrc, size);
544 EXPORT_SYMBOL(dma_memcpy);
547 * dma_memcpy_nocache - DMA memcpy under mutex lock
548 * - No cache flush/invalidate
550 * Do not check arguments before starting the DMA memcpy. Break the transfer
551 * up into two pieces. The first transfer is in multiples of 64k and the
552 * second transfer is the piece smaller than 64k.
554 void *dma_memcpy_nocache(void *pdst, const void *psrc, size_t size)
556 #ifdef DMA_MMR_SIZE_32
557 _dma_memcpy(pdst, psrc, size);
561 bulk = size & ~0xffff;
564 _dma_memcpy(pdst, psrc, bulk);
565 _dma_memcpy(pdst + bulk, psrc + bulk, rest);
569 EXPORT_SYMBOL(dma_memcpy_nocache);
572 * safe_dma_memcpy - DMA memcpy w/argument checking
574 * Verify arguments are safe before heading to dma_memcpy().
576 void *safe_dma_memcpy(void *dst, const void *src, size_t size)
578 if (!access_ok(VERIFY_WRITE, dst, size))
580 if (!access_ok(VERIFY_READ, src, size))
582 return dma_memcpy(dst, src, size);
584 EXPORT_SYMBOL(safe_dma_memcpy);
586 static void _dma_out(unsigned long addr, unsigned long buf, unsigned DMA_MMR_SIZE_TYPE len,
587 u16 size, u16 dma_size)
589 blackfin_dcache_flush_range(buf, buf + len * size);
590 __dma_memcpy(addr, 0, buf, size, len, dma_size);
593 static void _dma_in(unsigned long addr, unsigned long buf, unsigned DMA_MMR_SIZE_TYPE len,
594 u16 size, u16 dma_size)
596 blackfin_dcache_invalidate_range(buf, buf + len * size);
597 __dma_memcpy(buf, size, addr, 0, len, dma_size);
600 #define MAKE_DMA_IO(io, bwl, isize, dmasize, cnst) \
601 void dma_##io##s##bwl(unsigned long addr, cnst void *buf, unsigned DMA_MMR_SIZE_TYPE len) \
603 _dma_##io(addr, (unsigned long)buf, len, isize, WDSIZE_##dmasize); \
605 EXPORT_SYMBOL(dma_##io##s##bwl)
606 MAKE_DMA_IO(out, b, 1, 8, const);
607 MAKE_DMA_IO(in, b, 1, 8, );
608 MAKE_DMA_IO(out, w, 2, 16, const);
609 MAKE_DMA_IO(in, w, 2, 16, );
610 MAKE_DMA_IO(out, l, 4, 32, const);
611 MAKE_DMA_IO(in, l, 4, 32, );