arm64: Implement custom mmap functions for dma mapping

[firefly-linux-kernel-4.4.55.git] / arch / arm64 / mm / dma-mapping.c

diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c
index ba0ff75666ee08c7060c34026bd8a6a1a9fd596b..18b83177f4b63341e84e793bcc6ea11b3f5f5817 100644 (file)
--- a/arch/arm64/mm/dma-mapping.c
+++ b/arch/arm64/mm/dma-mapping.c
@@ -21,6 +21,7 @@
 #include <linux/export.h>
 #include <linux/slab.h>
 #include <linux/dma-mapping.h>
+#include <linux/dma-contiguous.h>
 #include <linux/vmalloc.h>
 #include <linux/swiotlb.h>
 
@@ -33,17 +34,43 @@ static void *__dma_alloc_coherent(struct device *dev, size_t size,
                                  dma_addr_t *dma_handle, gfp_t flags,
                                  struct dma_attrs *attrs)
 {
-       if (IS_ENABLED(CONFIG_ZONE_DMA32) &&
+       if (IS_ENABLED(CONFIG_ZONE_DMA) &&
            dev->coherent_dma_mask <= DMA_BIT_MASK(32))
-               flags |= GFP_DMA32;
-       return swiotlb_alloc_coherent(dev, size, dma_handle, flags);
+               flags |= GFP_DMA;
+       if (IS_ENABLED(CONFIG_DMA_CMA)) {
+               struct page *page;
+
+               size = PAGE_ALIGN(size);
+               page = dma_alloc_from_contiguous(dev, size >> PAGE_SHIFT,
+                                                       get_order(size));
+               if (!page)
+                       return NULL;
+
+               *dma_handle = phys_to_dma(dev, page_to_phys(page));
+               return page_address(page);
+       } else {
+               return swiotlb_alloc_coherent(dev, size, dma_handle, flags);
+       }
 }
 
 static void __dma_free_coherent(struct device *dev, size_t size,
                                void *vaddr, dma_addr_t dma_handle,
                                struct dma_attrs *attrs)
 {
-       swiotlb_free_coherent(dev, size, vaddr, dma_handle);
+       if (dev == NULL) {
+               WARN_ONCE(1, "Use an actual device structure for DMA allocation\n");
+               return;
+       }
+
+       if (IS_ENABLED(CONFIG_DMA_CMA)) {
+               phys_addr_t paddr = dma_to_phys(dev, dma_handle);
+
+               dma_release_from_contiguous(dev,
+                                       phys_to_page(paddr),
+                                       size >> PAGE_SHIFT);
+       } else {
+               swiotlb_free_coherent(dev, size, vaddr, dma_handle);
+       }
 }
 
 static void *__dma_alloc_noncoherent(struct device *dev, size_t size,
@@ -189,9 +216,52 @@ static void __swiotlb_sync_sg_for_device(struct device *dev,
                               sg->length, dir);
 }
 
+/* vma->vm_page_prot must be set appropriately before calling this function */
+static int __dma_common_mmap(struct device *dev, struct vm_area_struct *vma,
+                            void *cpu_addr, dma_addr_t dma_addr, size_t size)
+{
+       int ret = -ENXIO;
+       unsigned long nr_vma_pages = (vma->vm_end - vma->vm_start) >>
+                                       PAGE_SHIFT;
+       unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
+       unsigned long pfn = dma_to_phys(dev, dma_addr) >> PAGE_SHIFT;
+       unsigned long off = vma->vm_pgoff;
+
+       if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret))
+               return ret;
+
+       if (off < nr_pages && nr_vma_pages <= (nr_pages - off)) {
+               ret = remap_pfn_range(vma, vma->vm_start,
+                                     pfn + off,
+                                     vma->vm_end - vma->vm_start,
+                                     vma->vm_page_prot);
+       }
+
+       return ret;
+}
+
+static int __swiotlb_mmap_noncoherent(struct device *dev,
+               struct vm_area_struct *vma,
+               void *cpu_addr, dma_addr_t dma_addr, size_t size,
+               struct dma_attrs *attrs)
+{
+       vma->vm_page_prot = pgprot_dmacoherent(vma->vm_page_prot);
+       return __dma_common_mmap(dev, vma, cpu_addr, dma_addr, size);
+}
+
+static int __swiotlb_mmap_coherent(struct device *dev,
+               struct vm_area_struct *vma,
+               void *cpu_addr, dma_addr_t dma_addr, size_t size,
+               struct dma_attrs *attrs)
+{
+       /* Just use whatever page_prot attributes were specified */
+       return __dma_common_mmap(dev, vma, cpu_addr, dma_addr, size);
+}
+
 struct dma_map_ops noncoherent_swiotlb_dma_ops = {
        .alloc = __dma_alloc_noncoherent,
        .free = __dma_free_noncoherent,
+       .mmap = __swiotlb_mmap_noncoherent,
        .map_page = __swiotlb_map_page,
        .unmap_page = __swiotlb_unmap_page,
        .map_sg = __swiotlb_map_sg_attrs,
@@ -208,6 +278,7 @@ EXPORT_SYMBOL(noncoherent_swiotlb_dma_ops);
 struct dma_map_ops coherent_swiotlb_dma_ops = {
        .alloc = __dma_alloc_coherent,
        .free = __dma_free_coherent,
+       .mmap = __swiotlb_mmap_coherent,
        .map_page = swiotlb_map_page,
        .unmap_page = swiotlb_unmap_page,
        .map_sg = swiotlb_map_sg_attrs,
@@ -221,11 +292,17 @@ struct dma_map_ops coherent_swiotlb_dma_ops = {
 };
 EXPORT_SYMBOL(coherent_swiotlb_dma_ops);
 
-void __init arm64_swiotlb_init(void)
+extern int swiotlb_late_init_with_default_size(size_t default_size);
+
+static int __init swiotlb_late_init(void)
 {
-       dma_ops = &coherent_swiotlb_dma_ops;
-       swiotlb_init(1);
+       size_t swiotlb_size = min(SZ_64M, MAX_ORDER_NR_PAGES << PAGE_SHIFT);
+
+       dma_ops = &noncoherent_swiotlb_dma_ops;
+
+       return swiotlb_late_init_with_default_size(swiotlb_size);
 }
+subsys_initcall(swiotlb_late_init);
 
 #define PREALLOC_DMA_DEBUG_ENTRIES     4096