drivers/staging/android/ion/ion_system_heap.c

   1 /*
   2  * drivers/staging/android/ion/ion_system_heap.c
   3  *
   4  * Copyright (C) 2011 Google, Inc.
   5  *
   6  * This software is licensed under the terms of the GNU General Public
   7  * License version 2, as published by the Free Software Foundation, and
   8  * may be copied, distributed, and modified under those terms.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  */
  16
  17 #include <asm/page.h>
  18 #include <linux/dma-mapping.h>
  19 #include <linux/err.h>
  20 #include <linux/highmem.h>
  21 #include <linux/mm.h>
  22 #include <linux/scatterlist.h>
  23 #include <linux/slab.h>
  24 #include <linux/vmalloc.h>
  25 #include "ion.h"
  26 #include "ion_priv.h"
  27
  28 struct page_info {
  29         struct page *page;
  30         unsigned long order;
  31         struct list_head list;
  32 };
  33
  34 static struct page_info *alloc_largest_available(unsigned long size,
  35                                                  bool split_pages)
  36 {
  37         static unsigned int orders[] = {8, 4, 0};
  38         struct page *page;
  39         struct page_info *info;
  40         int i;
  41
  42         for (i = 0; i < ARRAY_SIZE(orders); i++) {
  43                 if (size < (1 << orders[i]) * PAGE_SIZE)
  44                         continue;
  45                 page = alloc_pages(GFP_HIGHUSER | __GFP_ZERO |
  46                                    __GFP_NOWARN | __GFP_NORETRY, orders[i]);
  47                 if (!page)
  48                         continue;
  49                 if (split_pages)
  50                         split_page(page, orders[i]);
  51                 info = kmalloc(sizeof(struct page_info *), GFP_KERNEL);
  52                 info->page = page;
  53                 info->order = orders[i];
  54                 return info;
  55         }
  56         return NULL;
  57 }
  58
  59 static int ion_system_heap_allocate(struct ion_heap *heap,
  60                                      struct ion_buffer *buffer,
  61                                      unsigned long size, unsigned long align,
  62                                      unsigned long flags)
  63 {
  64         struct sg_table *table;
  65         struct scatterlist *sg;
  66         int ret;
  67         struct list_head pages;
  68         struct page_info *info, *tmp_info;
  69         int i = 0;
  70         long size_remaining = PAGE_ALIGN(size);
  71         bool split_pages = ion_buffer_fault_user_mappings(buffer);
  72
  73
  74         INIT_LIST_HEAD(&pages);
  75         while (size_remaining > 0) {
  76                 info = alloc_largest_available(size_remaining, split_pages);
  77                 if (!info)
  78                         goto err;
  79                 list_add_tail(&info->list, &pages);
  80                 size_remaining -= (1 << info->order) * PAGE_SIZE;
  81                 i++;
  82         }
  83
  84         table = kmalloc(sizeof(struct sg_table), GFP_KERNEL);
  85         if (!table)
  86                 goto err;
  87
  88         if (split_pages)
  89                 ret = sg_alloc_table(table, PAGE_ALIGN(size) / PAGE_SIZE,
  90                                      GFP_KERNEL);
  91         else
  92                 ret = sg_alloc_table(table, i, GFP_KERNEL);
  93
  94         if (ret)
  95                 goto err1;
  96
  97         sg = table->sgl;
  98         list_for_each_entry_safe(info, tmp_info, &pages, list) {
  99                 struct page *page = info->page;
 100
 101                 if (split_pages) {
 102                         for (i = 0; i < (1 << info->order); i++) {
 103                                 sg_set_page(sg, page + i, PAGE_SIZE, 0);
 104                                 sg = sg_next(sg);
 105                         }
 106                 } else {
 107                         sg_set_page(sg, page, (1 << info->order) * PAGE_SIZE,
 108                                     0);
 109                         sg = sg_next(sg);
 110                 }
 111                 list_del(&info->list);
 112                 kfree(info);
 113         }
 114
 115         dma_sync_sg_for_device(NULL, table->sgl, table->nents,
 116                                DMA_BIDIRECTIONAL);
 117
 118         buffer->priv_virt = table;
 119         return 0;
 120 err1:
 121         kfree(table);
 122 err:
 123         list_for_each_entry(info, &pages, list) {
 124                 if (split_pages)
 125                         for (i = 0; i < (1 << info->order); i++)
 126                                 __free_page(info->page + i);
 127                 else
 128                         __free_pages(info->page, info->order);
 129
 130                 kfree(info);
 131         }
 132         return -ENOMEM;
 133 }
 134
 135 void ion_system_heap_free(struct ion_buffer *buffer)
 136 {
 137         int i;
 138         struct scatterlist *sg;
 139         struct sg_table *table = buffer->priv_virt;
 140
 141         for_each_sg(table->sgl, sg, table->nents, i)
 142                 __free_pages(sg_page(sg), get_order(sg_dma_len(sg)));
 143         if (buffer->sg_table)
 144                 sg_free_table(buffer->sg_table);
 145         kfree(buffer->sg_table);
 146 }
 147
 148 struct sg_table *ion_system_heap_map_dma(struct ion_heap *heap,
 149                                          struct ion_buffer *buffer)
 150 {
 151         return buffer->priv_virt;
 152 }
 153
 154 void ion_system_heap_unmap_dma(struct ion_heap *heap,
 155                                struct ion_buffer *buffer)
 156 {
 157         return;
 158 }
 159
 160 void *ion_system_heap_map_kernel(struct ion_heap *heap,
 161                                  struct ion_buffer *buffer)
 162 {
 163         struct scatterlist *sg;
 164         int i, j;
 165         void *vaddr;
 166         pgprot_t pgprot;
 167         struct sg_table *table = buffer->priv_virt;
 168         int npages = PAGE_ALIGN(buffer->size) / PAGE_SIZE;
 169         struct page **pages = kzalloc(sizeof(struct page *) * npages,
 170                                      GFP_KERNEL);
 171         struct page **tmp = pages;
 172
 173         if (buffer->flags & ION_FLAG_CACHED)
 174                 pgprot = PAGE_KERNEL;
 175         else
 176                 pgprot = pgprot_writecombine(PAGE_KERNEL);
 177
 178         for_each_sg(table->sgl, sg, table->nents, i) {
 179                 int npages_this_entry = PAGE_ALIGN(sg_dma_len(sg)) / PAGE_SIZE;
 180                 struct page *page = sg_page(sg);
 181                 BUG_ON(i >= npages);
 182                 for (j = 0; j < npages_this_entry; j++) {
 183                         *(tmp++) = page++;
 184                 }
 185         }
 186         vaddr = vmap(pages, npages, VM_MAP, pgprot);
 187         kfree(pages);
 188
 189         return vaddr;
 190 }
 191
 192 void ion_system_heap_unmap_kernel(struct ion_heap *heap,
 193                                   struct ion_buffer *buffer)
 194 {
 195         vunmap(buffer->vaddr);
 196 }
 197
 198 int ion_system_heap_map_user(struct ion_heap *heap, struct ion_buffer *buffer,
 199                              struct vm_area_struct *vma)
 200 {
 201         struct sg_table *table = buffer->priv_virt;
 202         unsigned long addr = vma->vm_start;
 203         unsigned long offset = vma->vm_pgoff;
 204         struct scatterlist *sg;
 205         int i;
 206
 207         for_each_sg(table->sgl, sg, table->nents, i) {
 208                 if (offset) {
 209                         offset--;
 210                         continue;
 211                 }
 212                 remap_pfn_range(vma, addr, page_to_pfn(sg_page(sg)),
 213                                 sg_dma_len(sg), vma->vm_page_prot);
 214                 addr += sg_dma_len(sg);
 215                 if (addr >= vma->vm_end)
 216                         return 0;
 217         }
 218         return 0;
 219 }
 220
 221 static struct ion_heap_ops vmalloc_ops = {
 222         .allocate = ion_system_heap_allocate,
 223         .free = ion_system_heap_free,
 224         .map_dma = ion_system_heap_map_dma,
 225         .unmap_dma = ion_system_heap_unmap_dma,
 226         .map_kernel = ion_system_heap_map_kernel,
 227         .unmap_kernel = ion_system_heap_unmap_kernel,
 228         .map_user = ion_system_heap_map_user,
 229 };
 230
 231 struct ion_heap *ion_system_heap_create(struct ion_platform_heap *unused)
 232 {
 233         struct ion_heap *heap;
 234
 235         heap = kzalloc(sizeof(struct ion_heap), GFP_KERNEL);
 236         if (!heap)
 237                 return ERR_PTR(-ENOMEM);
 238         heap->ops = &vmalloc_ops;
 239         heap->type = ION_HEAP_TYPE_SYSTEM;
 240         return heap;
 241 }
 242
 243 void ion_system_heap_destroy(struct ion_heap *heap)
 244 {
 245         kfree(heap);
 246 }
 247
 248 static int ion_system_contig_heap_allocate(struct ion_heap *heap,
 249                                            struct ion_buffer *buffer,
 250                                            unsigned long len,
 251                                            unsigned long align,
 252                                            unsigned long flags)
 253 {
 254         buffer->priv_virt = kzalloc(len, GFP_KERNEL);
 255         if (!buffer->priv_virt)
 256                 return -ENOMEM;
 257         return 0;
 258 }
 259
 260 void ion_system_contig_heap_free(struct ion_buffer *buffer)
 261 {
 262         kfree(buffer->priv_virt);
 263 }
 264
 265 static int ion_system_contig_heap_phys(struct ion_heap *heap,
 266                                        struct ion_buffer *buffer,
 267                                        ion_phys_addr_t *addr, size_t *len)
 268 {
 269         *addr = virt_to_phys(buffer->priv_virt);
 270         *len = buffer->size;
 271         return 0;
 272 }
 273
 274 struct sg_table *ion_system_contig_heap_map_dma(struct ion_heap *heap,
 275                                                 struct ion_buffer *buffer)
 276 {
 277         struct sg_table *table;
 278         int ret;
 279
 280         table = kzalloc(sizeof(struct sg_table), GFP_KERNEL);
 281         if (!table)
 282                 return ERR_PTR(-ENOMEM);
 283         ret = sg_alloc_table(table, 1, GFP_KERNEL);
 284         if (ret) {
 285                 kfree(table);
 286                 return ERR_PTR(ret);
 287         }
 288         sg_set_page(table->sgl, virt_to_page(buffer->priv_virt), buffer->size,
 289                     0);
 290         return table;
 291 }
 292
 293 void ion_system_contig_heap_unmap_dma(struct ion_heap *heap,
 294                                       struct ion_buffer *buffer)
 295 {
 296         sg_free_table(buffer->sg_table);
 297         kfree(buffer->sg_table);
 298 }
 299
 300 int ion_system_contig_heap_map_user(struct ion_heap *heap,
 301                                     struct ion_buffer *buffer,
 302                                     struct vm_area_struct *vma)
 303 {
 304         unsigned long pfn = __phys_to_pfn(virt_to_phys(buffer->priv_virt));
 305         return remap_pfn_range(vma, vma->vm_start, pfn + vma->vm_pgoff,
 306                                vma->vm_end - vma->vm_start,
 307                                vma->vm_page_prot);
 308
 309 }
 310
 311 static struct ion_heap_ops kmalloc_ops = {
 312         .allocate = ion_system_contig_heap_allocate,
 313         .free = ion_system_contig_heap_free,
 314         .phys = ion_system_contig_heap_phys,
 315         .map_dma = ion_system_contig_heap_map_dma,
 316         .unmap_dma = ion_system_contig_heap_unmap_dma,
 317         .map_kernel = ion_system_heap_map_kernel,
 318         .unmap_kernel = ion_system_heap_unmap_kernel,
 319         .map_user = ion_system_contig_heap_map_user,
 320 };
 321
 322 struct ion_heap *ion_system_contig_heap_create(struct ion_platform_heap *unused)
 323 {
 324         struct ion_heap *heap;
 325
 326         heap = kzalloc(sizeof(struct ion_heap), GFP_KERNEL);
 327         if (!heap)
 328                 return ERR_PTR(-ENOMEM);
 329         heap->ops = &kmalloc_ops;
 330         heap->type = ION_HEAP_TYPE_SYSTEM_CONTIG;
 331         return heap;
 332 }
 333
 334 void ion_system_contig_heap_destroy(struct ion_heap *heap)
 335 {
 336         kfree(heap);
 337 }
 338