Say Y here if you are using a Rockchip SoC that includes an IOMMU
device.
+config RK_IOMMU
+ bool "RK IOMMU Support"
+ depends on ARCH_ROCKCHIP || COMPILE_TEST
+ select IOMMU_API
+ help
+ Support for IOMMUs found on Rockchip rk32xx,rk313x,rk33xx SOCs.
+ These IOMMUs allow virtualization of the address space used by most
+ cores within the multimedia subsystem.
+ Say Y here if you are using a Rockchip SoC that includes an IOMMU
+ device.
+
+config RK_IOVMM
+ bool "IO Virtual Memory Manager for RK IOMMUs"
+ select GENERIC_ALLOCATOR
+ depends on RK_IOMMU
+ help
+ Supporting the users of Rockchip IOMMU for allocating and mapping
+ an IO virtual memory region with a physical memory region
+ and managing the allocated virtual memory regions.
+
+config RK_IOMMU_DEBUG
+ bool "Debugging log for RK IOMMU"
+ depends on RK_IOMMU
+ help
+ Select this to see the detailed log message that shows what
+ happens in the IOMMU driver
+
+ Say N unless you need kernel log message for IOMMU debugging
+
config TEGRA_IOMMU_GART
bool "Tegra GART IOMMU Support"
depends on ARCH_TEGRA_2x_SOC
obj-$(CONFIG_OMAP_IOMMU) += omap-iommu.o
obj-$(CONFIG_OMAP_IOMMU_DEBUG) += omap-iommu-debug.o
obj-$(CONFIG_ROCKCHIP_IOMMU) += rockchip-iommu.o
+obj-$(CONFIG_RK_IOMMU) += rk-iommu.o
+obj-$(CONFIG_RK_IOVMM) += rk-iovmm.o
obj-$(CONFIG_TEGRA_IOMMU_GART) += tegra-gart.o
obj-$(CONFIG_TEGRA_IOMMU_SMMU) += tegra-smmu.o
obj-$(CONFIG_EXYNOS_IOMMU) += exynos-iommu.o
#include <linux/rockchip/cpu.h>
#include <linux/rockchip/iomap.h>
#include <linux/device.h>
-#include "rockchip-iommu.h"
+#include "rk-iommu.h"
/* We does not consider super section mapping (16MB) */
#define SPAGE_ORDER 12
short *lv2entcnt; /* free lv2 entry counter for each section */
spinlock_t lock; /* lock for this structure */
spinlock_t pgtablelock; /* lock for modifying page table @ pgtable */
+ struct iommu_domain domain;
};
+static struct rk_iommu_domain *to_rk_domain(struct iommu_domain *dom)
+{
+ return container_of(dom, struct rk_iommu_domain, domain);
+}
+
static bool rockchip_set_iommu_active(struct iommu_drvdata *data)
{
/* return true if the IOMMU was not active previously
report_iommu_fault(data->domain, data->iommu,
fault_address, flags);
if (data->fault_handler)
- data->fault_handler(data->iommu, IOMMU_PAGEFAULT, dte, fault_address, 1);
+ data->fault_handler(data->master, IOMMU_PAGEFAULT, dte, fault_address, 1);
rockchip_iommu_page_fault_done(data->res_bases[i],
data->dbgname);
static phys_addr_t rockchip_iommu_iova_to_phys(struct iommu_domain *domain,
dma_addr_t iova)
{
- struct rk_iommu_domain *priv = domain->priv;
+ struct rk_iommu_domain *priv = to_rk_domain(domain);
unsigned int *entry;
unsigned long flags;
phys_addr_t phys = 0;
static size_t rockchip_iommu_unmap(struct iommu_domain *domain,
unsigned long iova, size_t size)
{
- struct rk_iommu_domain *priv = domain->priv;
+ struct rk_iommu_domain *priv = to_rk_domain(domain);
unsigned long flags;
unsigned int *ent;
static int rockchip_iommu_map(struct iommu_domain *domain, unsigned long iova,
phys_addr_t paddr, size_t size, int prot)
{
- struct rk_iommu_domain *priv = domain->priv;
+ struct rk_iommu_domain *priv = to_rk_domain(domain);
unsigned int *entry;
unsigned long flags;
int ret = -ENOMEM;
static void rockchip_iommu_detach_device(struct iommu_domain *domain, struct device *dev)
{
struct iommu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
- struct rk_iommu_domain *priv = domain->priv;
+ struct rk_iommu_domain *priv = to_rk_domain(domain);
struct list_head *pos;
unsigned long flags;
bool found = false;
static int rockchip_iommu_attach_device(struct iommu_domain *domain, struct device *dev)
{
struct iommu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
- struct rk_iommu_domain *priv = domain->priv;
+ struct rk_iommu_domain *priv = to_rk_domain(domain);
unsigned long flags;
int ret;
BUG_ON(!list_empty(&data->node));
list_add_tail(&data->node, &priv->clients);
data->domain = domain;
+ data->master = dev;
}
spin_unlock_irqrestore(&priv->lock, flags);
dev_dbg(dev->archdata.iommu,"%s: IOMMU with pgtable 0x%x already attached\n",
__func__, (unsigned int)virt_to_phys(priv->pgtable));
} else {
- if (!(strstr(data->dbgname, "vpu") || strstr(data->dbgname, "hevc")))
+ if (!(strstr(data->dbgname, "vpu") ||
+ strstr(data->dbgname, "hevc") ||
+ strstr(data->dbgname, "vdec")))
dev_info(dev->archdata.iommu,"%s: Attached new IOMMU with pgtable 0x%x\n",
__func__, (unsigned int)virt_to_phys(priv->pgtable));
}
return ret;
}
-static void rockchip_iommu_domain_destroy(struct iommu_domain *domain)
+static void rockchip_iommu_domain_free(struct iommu_domain *domain)
{
- struct rk_iommu_domain *priv = domain->priv;
+ struct rk_iommu_domain *priv = to_rk_domain(domain);
int i;
WARN_ON(!list_empty(&priv->clients));
free_pages((unsigned long)priv->pgtable, 0);
free_pages((unsigned long)priv->lv2entcnt, 0);
- kfree(domain->priv);
- domain->priv = NULL;
+ kfree(priv);
}
-static int rockchip_iommu_domain_init(struct iommu_domain *domain)
+static struct iommu_domain *rockchip_iommu_domain_alloc(unsigned type)
{
struct rk_iommu_domain *priv;
+ if (type != IOMMU_DOMAIN_UNMANAGED)
+ return NULL;
+
priv = kzalloc(sizeof(*priv), GFP_KERNEL);
if (!priv)
- return -ENOMEM;
+ return NULL;
/*rk32xx iommu use 2 level pagetable,
level1 and leve2 both have 1024 entries,each entry occupy 4 bytes,
spin_lock_init(&priv->pgtablelock);
INIT_LIST_HEAD(&priv->clients);
- domain->priv = priv;
- return 0;
+ return &priv->domain;
err_counter:
free_pages((unsigned long)priv->pgtable, 0);
err_pgtable:
kfree(priv);
- return -ENOMEM;
+ return NULL;
}
static struct iommu_ops rk_iommu_ops = {
- .domain_init = rockchip_iommu_domain_init,
- .domain_destroy = rockchip_iommu_domain_destroy,
+ .domain_alloc = rockchip_iommu_domain_alloc,
+ .domain_free = rockchip_iommu_domain_free,
.attach_dev = rockchip_iommu_attach_device,
.detach_dev = rockchip_iommu_detach_device,
.map = rockchip_iommu_map,
{ .compatible = VPU_IOMMU_COMPATIBLE_NAME},
{ .compatible = ISP_IOMMU_COMPATIBLE_NAME},
{ .compatible = VOP_IOMMU_COMPATIBLE_NAME},
+ { .compatible = VDEC_IOMMU_COMPATIBLE_NAME},
{ /* end */ }
};
--- /dev/null
+/*
+ * Data structure definition for Rockchip IOMMU driver
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#ifndef __ASM_PLAT_IOMMU_H
+#define __ASM_PLAT_IOMMU_H
+
+#include <linux/kernel.h>
+#include <linux/spinlock.h>
+#include <linux/list.h>
+#include <linux/device.h>
+#include <linux/platform_device.h>
+#include <linux/genalloc.h>
+#include <linux/iommu.h>
+
+#include <linux/rockchip-iovmm.h>
+
+
+struct rk_iovmm {
+ struct iommu_domain *domain; /* iommu domain for this iovmm */
+ struct gen_pool *vmm_pool;
+ struct list_head regions_list; /* list of rk_vm_region */
+ spinlock_t lock; /* lock for updating regions_list */
+};
+
+struct iommu_drvdata {
+ struct list_head node; /* entry of rk_iommu_domain.clients */
+ struct device *iommu; /* IOMMU's device descriptor */
+ struct device *master; /* IOMMU's master device descriptor */
+ int num_res_mem;
+ int num_res_irq;
+ const char *dbgname;
+ void __iomem **res_bases;
+ int activations;
+ spinlock_t data_lock;
+ struct iommu_domain *domain; /* domain given to iommu_attach_device() */
+ unsigned int pgtable;
+ struct rk_iovmm vmm;
+ rockchip_iommu_fault_handler_t fault_handler;
+};
+
+#ifdef CONFIG_RK_IOVMM
+
+#define IOVA_START 0x10000000
+#define IOVM_SIZE (SZ_1G - SZ_4K) /* last 4K is for error values */
+
+struct rk_vm_region {
+ struct list_head node;
+ unsigned int start;
+ unsigned int size;
+};
+
+static inline struct rk_iovmm *rockchip_get_iovmm(struct device *dev)
+{
+ struct iommu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
+
+ BUG_ON(!dev->archdata.iommu || !data);
+
+ return &data->vmm;
+}
+
+int rockchip_init_iovmm(struct device *iommu, struct rk_iovmm *vmm);
+#else
+static inline int rockchip_init_iovmm(struct device *iommu,
+ struct rk_iovmm *vmm)
+{
+ return -ENOSYS;
+}
+#endif
+
+
+#ifdef CONFIG_RK_IOMMU
+
+/**
+ * rockchip_iommu_tlb_invalidate() - flush all TLB entry in iommu
+ * @owner: The device whose IOMMU.
+ *
+ * This function flush all TLB entry in iommu
+ */
+int rockchip_iommu_tlb_invalidate(struct device *owner);
+int rockchip_iommu_tlb_invalidate_global(struct device *owner);
+
+#else /* CONFIG_RK_IOMMU */
+static inline int rockchip_iommu_tlb_invalidate(struct device *owner)
+{
+ return -1;
+}
+static int rockchip_iommu_tlb_invalidate_global(struct device *owner)
+{
+ return -1;
+}
+
+#endif
+
+#endif /*__ASM_PLAT_IOMMU_H*/
--- /dev/null
+/*
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifdef CONFIG_ROCKCHIP_IOMMU_DEBUG
+#define DEBUG
+#endif
+
+#include <linux/kernel.h>
+#include <linux/hardirq.h>
+#include <linux/slab.h>
+#include <linux/scatterlist.h>
+#include <linux/err.h>
+
+#include <linux/of.h>
+#include <linux/of_platform.h>
+
+#include "rk-iommu.h"
+
+#define IOMMU_REGION_GUARD (2<<PAGE_SHIFT)
+
+static struct rk_vm_region *find_region(struct rk_iovmm *vmm, dma_addr_t iova)
+{
+ struct rk_vm_region *region;
+
+ list_for_each_entry(region, &vmm->regions_list, node)
+ if (region->start == iova)
+ return region;
+
+ return NULL;
+}
+
+int rockchip_iovmm_invalidate_tlb(struct device *dev)
+{
+ int ret = rockchip_iommu_tlb_invalidate_global(dev);
+
+ return ret;
+}
+
+void rockchip_iovmm_set_fault_handler(struct device *dev,
+ rockchip_iommu_fault_handler_t handler)
+{
+ struct iommu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
+
+ data->fault_handler = handler;
+}
+
+int rockchip_iovmm_activate(struct device *dev)
+{
+ struct rk_iovmm *vmm = rockchip_get_iovmm(dev);
+
+ return iommu_attach_device(vmm->domain, dev);
+}
+
+void rockchip_iovmm_deactivate(struct device *dev)
+{
+ struct rk_iovmm *vmm = rockchip_get_iovmm(dev);
+
+ iommu_detach_device(vmm->domain, dev);
+}
+
+dma_addr_t rockchip_iovmm_map(struct device *dev,
+ struct scatterlist *sg, off_t offset, size_t size)
+{
+ off_t start_off;
+ dma_addr_t addr, start = 0;
+ size_t mapped_size = 0;
+ struct rk_vm_region *region;
+ struct rk_iovmm *vmm = rockchip_get_iovmm(dev);
+ int order;
+ int ret;
+
+ for (; sg_dma_len(sg) < offset; sg = sg_next(sg))
+ offset -= sg_dma_len(sg);
+
+ start_off = offset_in_page(sg_phys(sg) + offset);
+ size = PAGE_ALIGN(size + start_off);
+
+ order = __fls(min_t(size_t, size, SZ_1M));
+
+ region = kmalloc(sizeof(*region), GFP_KERNEL);
+ if (!region) {
+ ret = -ENOMEM;
+ goto err_map_nomem;
+ }
+
+ start = (dma_addr_t)gen_pool_alloc(vmm->vmm_pool,
+ size+IOMMU_REGION_GUARD);
+ if (!start) {
+ ret = -ENOMEM;
+ goto err_map_noiomem;
+ }
+
+ pr_debug("%s: size = %zx\n", __func__, size);
+
+ addr = start;
+ do {
+ phys_addr_t phys;
+ size_t len;
+
+ phys = sg_phys(sg);
+ len = sg_dma_len(sg);
+
+ /* if back to back sg entries are contiguous consolidate them */
+ while (sg_next(sg) && sg_phys(sg) +
+ sg_dma_len(sg) == sg_phys(sg_next(sg))) {
+ len += sg_dma_len(sg_next(sg));
+ sg = sg_next(sg);
+ }
+
+ if (offset > 0) {
+ len -= offset;
+ phys += offset;
+ offset = 0;
+ }
+
+ if (offset_in_page(phys)) {
+ len += offset_in_page(phys);
+ phys = round_down(phys, PAGE_SIZE);
+ }
+
+ len = PAGE_ALIGN(len);
+
+ if (len > (size - mapped_size))
+ len = size - mapped_size;
+ pr_debug("addr = %pad, phys = %pa, len = %zx\n", &addr, &phys, len);
+ ret = iommu_map(vmm->domain, addr, phys, len, 0);
+ if (ret)
+ break;
+
+ addr += len;
+ mapped_size += len;
+ } while ((sg = sg_next(sg)) && (mapped_size < size));
+
+ BUG_ON(mapped_size > size);
+
+ if (mapped_size < size)
+ goto err_map_map;
+
+ region->start = start + start_off;
+ region->size = size;
+
+ INIT_LIST_HEAD(®ion->node);
+
+ spin_lock(&vmm->lock);
+
+ list_add(®ion->node, &vmm->regions_list);
+
+ spin_unlock(&vmm->lock);
+
+ ret = rockchip_iommu_tlb_invalidate(dev);
+ if (ret) {
+ spin_lock(&vmm->lock);
+ list_del(®ion->node);
+ spin_unlock(&vmm->lock);
+ goto err_map_map;
+ }
+ dev_dbg(dev->archdata.iommu, "IOVMM: Allocated VM region @ %p/%#X bytes.\n",
+ ®ion->start, region->size);
+
+ return region->start;
+
+err_map_map:
+ iommu_unmap(vmm->domain, start, mapped_size);
+ gen_pool_free(vmm->vmm_pool, start, size);
+err_map_noiomem:
+ kfree(region);
+err_map_nomem:
+ dev_err(dev->archdata.iommu, "IOVMM: Failed to allocated VM region for %zx bytes.\n", size);
+ return (dma_addr_t)ret;
+}
+
+void rockchip_iovmm_unmap(struct device *dev, dma_addr_t iova)
+{
+ struct rk_vm_region *region;
+ struct rk_iovmm *vmm = rockchip_get_iovmm(dev);
+ size_t unmapped_size;
+
+ /* This function must not be called in IRQ handlers */
+ BUG_ON(in_irq());
+
+ spin_lock(&vmm->lock);
+
+ region = find_region(vmm, iova);
+ if (WARN_ON(!region)) {
+ spin_unlock(&vmm->lock);
+ return;
+ }
+
+ list_del(®ion->node);
+
+ spin_unlock(&vmm->lock);
+
+ region->start = round_down(region->start, PAGE_SIZE);
+
+ unmapped_size = iommu_unmap(vmm->domain,
+ region->start, region->size);
+ /*
+ rockchip_iommu_tlb_invalidate(dev);
+ */
+ gen_pool_free(vmm->vmm_pool, region->start,
+ region->size+IOMMU_REGION_GUARD);
+
+ WARN_ON(unmapped_size != region->size);
+
+ dev_dbg(dev->archdata.iommu, "IOVMM: Unmapped %zx bytes from %pad.\n",
+ unmapped_size, ®ion->start);
+
+ kfree(region);
+}
+
+int rockchip_iovmm_map_oto(struct device *dev, phys_addr_t phys, size_t size)
+{
+ struct rk_vm_region *region;
+ struct rk_iovmm *vmm = rockchip_get_iovmm(dev);
+ int ret;
+
+ if (WARN_ON((phys + size) >= IOVA_START)) {
+ dev_err(dev->archdata.iommu, "Unable to create one to one mapping for %zx @ %pa\n",
+ size, &phys);
+ return -EINVAL;
+ }
+
+ region = kmalloc(sizeof(*region), GFP_KERNEL);
+ if (!region)
+ return -ENOMEM;
+
+ if (WARN_ON(phys & ~PAGE_MASK))
+ phys = round_down(phys, PAGE_SIZE);
+
+
+ ret = iommu_map(vmm->domain, (dma_addr_t)phys, phys, size, 0);
+ if (ret < 0) {
+ kfree(region);
+ return ret;
+ }
+
+ region->start = (dma_addr_t)phys;
+ region->size = size;
+ INIT_LIST_HEAD(®ion->node);
+
+ spin_lock(&vmm->lock);
+
+ list_add(®ion->node, &vmm->regions_list);
+
+ spin_unlock(&vmm->lock);
+
+ ret = rockchip_iommu_tlb_invalidate(dev);
+ if (ret)
+ return ret;
+
+ return 0;
+}
+
+void rockchip_iovmm_unmap_oto(struct device *dev, phys_addr_t phys)
+{
+ struct rk_vm_region *region;
+ struct rk_iovmm *vmm = rockchip_get_iovmm(dev);
+ size_t unmapped_size;
+
+ /* This function must not be called in IRQ handlers */
+ BUG_ON(in_irq());
+
+ if (WARN_ON(phys & ~PAGE_MASK))
+ phys = round_down(phys, PAGE_SIZE);
+
+ spin_lock(&vmm->lock);
+
+ region = find_region(vmm, (dma_addr_t)phys);
+ if (WARN_ON(!region)) {
+ spin_unlock(&vmm->lock);
+ return;
+ }
+
+ list_del(®ion->node);
+
+ spin_unlock(&vmm->lock);
+
+ unmapped_size = iommu_unmap(vmm->domain, region->start, region->size);
+ WARN_ON(unmapped_size != region->size);
+ dev_dbg(dev->archdata.iommu, "IOVMM: Unmapped %zx bytes from %pad.\n",
+ unmapped_size, ®ion->start);
+
+ kfree(region);
+}
+
+int rockchip_init_iovmm(struct device *iommu, struct rk_iovmm *vmm)
+{
+ int ret = 0;
+
+ vmm->vmm_pool = gen_pool_create(PAGE_SHIFT, -1);
+ if (!vmm->vmm_pool) {
+ ret = -ENOMEM;
+ goto err_setup_genalloc;
+ }
+
+ /* (1GB - 4KB) addr space from 0x10000000 */
+ ret = gen_pool_add(vmm->vmm_pool, IOVA_START, IOVM_SIZE, -1);
+ if (ret)
+ goto err_setup_domain;
+
+ vmm->domain = iommu_domain_alloc(&platform_bus_type);
+ if (!vmm->domain) {
+ ret = -ENOMEM;
+ goto err_setup_domain;
+ }
+
+ spin_lock_init(&vmm->lock);
+
+ INIT_LIST_HEAD(&vmm->regions_list);
+
+ dev_info(iommu, "IOVMM: Created %#x B IOVMM from %#x.\n",
+ IOVM_SIZE, IOVA_START);
+ return 0;
+err_setup_domain:
+ gen_pool_destroy(vmm->vmm_pool);
+err_setup_genalloc:
+ dev_err(iommu, "IOVMM: Failed to create IOVMM (%d)\n", ret);
+
+ return ret;
+}
+++ /dev/null
-/*
- * Data structure definition for Rockchip IOMMU driver
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-#ifndef __ASM_PLAT_IOMMU_H
-#define __ASM_PLAT_IOMMU_H
-
-#include <linux/kernel.h>
-#include <linux/spinlock.h>
-#include <linux/list.h>
-#include <linux/device.h>
-#include <linux/platform_device.h>
-#include <linux/genalloc.h>
-#include <linux/iommu.h>
-
-#include <linux/rockchip-iovmm.h>
-
-
-struct rk_iovmm {
- struct iommu_domain *domain; /* iommu domain for this iovmm */
- struct gen_pool *vmm_pool;
- struct list_head regions_list; /* list of rk_vm_region */
- spinlock_t lock; /* lock for updating regions_list */
-};
-
-struct iommu_drvdata {
- struct list_head node; /* entry of rk_iommu_domain.clients */
- struct device *iommu; /* IOMMU's device descriptor */
- int num_res_mem;
- int num_res_irq;
- const char *dbgname;
- void __iomem **res_bases;
- int activations;
- spinlock_t data_lock;
- struct iommu_domain *domain; /* domain given to iommu_attach_device() */
- unsigned int pgtable;
- struct rk_iovmm vmm;
- rockchip_iommu_fault_handler_t fault_handler;
-};
-
-#ifdef CONFIG_ROCKCHIP_IOVMM
-
-#define IOVA_START 0x10000000
-#define IOVM_SIZE (SZ_1G - SZ_4K) /* last 4K is for error values */
-
-struct rk_vm_region {
- struct list_head node;
- unsigned int start;
- unsigned int size;
-};
-
-static inline struct rk_iovmm *rockchip_get_iovmm(struct device *dev)
-{
- struct iommu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
-
- BUG_ON(!dev->archdata.iommu || !data);
-
- return &data->vmm;
-}
-
-int rockchip_init_iovmm(struct device *iommu, struct rk_iovmm *vmm);
-#else
-static inline int rockchip_init_iovmm(struct device *iommu,
- struct rk_iovmm *vmm)
-{
- return -ENOSYS;
-}
-#endif
-
-
-#ifdef CONFIG_ROCKCHIP_IOMMU
-
-/**
- * rockchip_iommu_tlb_invalidate() - flush all TLB entry in iommu
- * @owner: The device whose IOMMU.
- *
- * This function flush all TLB entry in iommu
- */
-int rockchip_iommu_tlb_invalidate(struct device *owner);
-int rockchip_iommu_tlb_invalidate_global(struct device *owner);
-
-#else /* CONFIG_ROCKCHIP_IOMMU */
-static inline int rockchip_iommu_tlb_invalidate(struct device *owner)
-{
- return -1;
-}
-static int rockchip_iommu_tlb_invalidate_global(struct device *owner)
-{
- return -1;
-}
-
-#endif
-
-#endif /*__ASM_PLAT_IOMMU_H*/
+++ /dev/null
-/*
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-#ifdef CONFIG_ROCKCHIP_IOMMU_DEBUG
-#define DEBUG
-#endif
-
-#include <linux/kernel.h>
-#include <linux/hardirq.h>
-#include <linux/slab.h>
-#include <linux/scatterlist.h>
-#include <linux/err.h>
-
-#include <linux/of.h>
-#include <linux/of_platform.h>
-
-#include "rockchip-iommu.h"
-
-#define IOMMU_REGION_GUARD (2<<PAGE_SHIFT)
-
-static struct rk_vm_region *find_region(struct rk_iovmm *vmm, dma_addr_t iova)
-{
- struct rk_vm_region *region;
-
- list_for_each_entry(region, &vmm->regions_list, node)
- if (region->start == iova)
- return region;
-
- return NULL;
-}
-
-int rockchip_iovmm_invalidate_tlb(struct device *dev)
-{
- int ret = rockchip_iommu_tlb_invalidate_global(dev);
-
- return ret;
-}
-
-void rockchip_iovmm_set_fault_handler(struct device *dev,
- rockchip_iommu_fault_handler_t handler)
-{
- struct iommu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
-
- data->fault_handler = handler;
-}
-
-int rockchip_iovmm_activate(struct device *dev)
-{
- struct rk_iovmm *vmm = rockchip_get_iovmm(dev);
-
- return iommu_attach_device(vmm->domain, dev);
-}
-
-void rockchip_iovmm_deactivate(struct device *dev)
-{
- struct rk_iovmm *vmm = rockchip_get_iovmm(dev);
-
- iommu_detach_device(vmm->domain, dev);
-}
-
-dma_addr_t rockchip_iovmm_map(struct device *dev,
- struct scatterlist *sg, off_t offset, size_t size)
-{
- off_t start_off;
- dma_addr_t addr, start = 0;
- size_t mapped_size = 0;
- struct rk_vm_region *region;
- struct rk_iovmm *vmm = rockchip_get_iovmm(dev);
- int order;
- int ret;
-
- for (; sg_dma_len(sg) < offset; sg = sg_next(sg))
- offset -= sg_dma_len(sg);
-
- start_off = offset_in_page(sg_phys(sg) + offset);
- size = PAGE_ALIGN(size + start_off);
-
- order = __fls(min_t(size_t, size, SZ_1M));
-
- region = kmalloc(sizeof(*region), GFP_KERNEL);
- if (!region) {
- ret = -ENOMEM;
- goto err_map_nomem;
- }
-
- start = (dma_addr_t)gen_pool_alloc(vmm->vmm_pool,
- size+IOMMU_REGION_GUARD);
- if (!start) {
- ret = -ENOMEM;
- goto err_map_noiomem;
- }
-
- pr_debug("%s: size = %zx\n", __func__, size);
-
- addr = start;
- do {
- phys_addr_t phys;
- size_t len;
-
- phys = sg_phys(sg);
- len = sg_dma_len(sg);
-
- /* if back to back sg entries are contiguous consolidate them */
- while (sg_next(sg) && sg_phys(sg) +
- sg_dma_len(sg) == sg_phys(sg_next(sg))) {
- len += sg_dma_len(sg_next(sg));
- sg = sg_next(sg);
- }
-
- if (offset > 0) {
- len -= offset;
- phys += offset;
- offset = 0;
- }
-
- if (offset_in_page(phys)) {
- len += offset_in_page(phys);
- phys = round_down(phys, PAGE_SIZE);
- }
-
- len = PAGE_ALIGN(len);
-
- if (len > (size - mapped_size))
- len = size - mapped_size;
- pr_debug("addr = %pad, phys = %pa, len = %zx\n", &addr, &phys, len);
- ret = iommu_map(vmm->domain, addr, phys, len, 0);
- if (ret)
- break;
-
- addr += len;
- mapped_size += len;
- } while ((sg = sg_next(sg)) && (mapped_size < size));
-
- BUG_ON(mapped_size > size);
-
- if (mapped_size < size)
- goto err_map_map;
-
- region->start = start + start_off;
- region->size = size;
-
- INIT_LIST_HEAD(®ion->node);
-
- spin_lock(&vmm->lock);
-
- list_add(®ion->node, &vmm->regions_list);
-
- spin_unlock(&vmm->lock);
-
- ret = rockchip_iommu_tlb_invalidate(dev);
- if (ret) {
- spin_lock(&vmm->lock);
- list_del(®ion->node);
- spin_unlock(&vmm->lock);
- goto err_map_map;
- }
- dev_dbg(dev->archdata.iommu, "IOVMM: Allocated VM region @ %p/%#X bytes.\n",
- ®ion->start, region->size);
-
- return region->start;
-
-err_map_map:
- iommu_unmap(vmm->domain, start, mapped_size);
- gen_pool_free(vmm->vmm_pool, start, size);
-err_map_noiomem:
- kfree(region);
-err_map_nomem:
- dev_err(dev->archdata.iommu, "IOVMM: Failed to allocated VM region for %zx bytes.\n", size);
- return (dma_addr_t)ret;
-}
-
-void rockchip_iovmm_unmap(struct device *dev, dma_addr_t iova)
-{
- struct rk_vm_region *region;
- struct rk_iovmm *vmm = rockchip_get_iovmm(dev);
- size_t unmapped_size;
-
- /* This function must not be called in IRQ handlers */
- BUG_ON(in_irq());
-
- spin_lock(&vmm->lock);
-
- region = find_region(vmm, iova);
- if (WARN_ON(!region)) {
- spin_unlock(&vmm->lock);
- return;
- }
-
- list_del(®ion->node);
-
- spin_unlock(&vmm->lock);
-
- region->start = round_down(region->start, PAGE_SIZE);
-
- unmapped_size = iommu_unmap(vmm->domain,
- region->start, region->size);
- /*
- rockchip_iommu_tlb_invalidate(dev);
- */
- gen_pool_free(vmm->vmm_pool, region->start,
- region->size+IOMMU_REGION_GUARD);
-
- WARN_ON(unmapped_size != region->size);
-
- dev_dbg(dev->archdata.iommu, "IOVMM: Unmapped %zx bytes from %pad.\n",
- unmapped_size, ®ion->start);
-
- kfree(region);
-}
-
-int rockchip_iovmm_map_oto(struct device *dev, phys_addr_t phys, size_t size)
-{
- struct rk_vm_region *region;
- struct rk_iovmm *vmm = rockchip_get_iovmm(dev);
- int ret;
-
- if (WARN_ON((phys + size) >= IOVA_START)) {
- dev_err(dev->archdata.iommu, "Unable to create one to one mapping for %zx @ %pa\n",
- size, &phys);
- return -EINVAL;
- }
-
- region = kmalloc(sizeof(*region), GFP_KERNEL);
- if (!region)
- return -ENOMEM;
-
- if (WARN_ON(phys & ~PAGE_MASK))
- phys = round_down(phys, PAGE_SIZE);
-
-
- ret = iommu_map(vmm->domain, (dma_addr_t)phys, phys, size, 0);
- if (ret < 0) {
- kfree(region);
- return ret;
- }
-
- region->start = (dma_addr_t)phys;
- region->size = size;
- INIT_LIST_HEAD(®ion->node);
-
- spin_lock(&vmm->lock);
-
- list_add(®ion->node, &vmm->regions_list);
-
- spin_unlock(&vmm->lock);
-
- ret = rockchip_iommu_tlb_invalidate(dev);
- if (ret)
- return ret;
-
- return 0;
-}
-
-void rockchip_iovmm_unmap_oto(struct device *dev, phys_addr_t phys)
-{
- struct rk_vm_region *region;
- struct rk_iovmm *vmm = rockchip_get_iovmm(dev);
- size_t unmapped_size;
-
- /* This function must not be called in IRQ handlers */
- BUG_ON(in_irq());
-
- if (WARN_ON(phys & ~PAGE_MASK))
- phys = round_down(phys, PAGE_SIZE);
-
- spin_lock(&vmm->lock);
-
- region = find_region(vmm, (dma_addr_t)phys);
- if (WARN_ON(!region)) {
- spin_unlock(&vmm->lock);
- return;
- }
-
- list_del(®ion->node);
-
- spin_unlock(&vmm->lock);
-
- unmapped_size = iommu_unmap(vmm->domain, region->start, region->size);
- WARN_ON(unmapped_size != region->size);
- dev_dbg(dev->archdata.iommu, "IOVMM: Unmapped %zx bytes from %pad.\n",
- unmapped_size, ®ion->start);
-
- kfree(region);
-}
-
-int rockchip_init_iovmm(struct device *iommu, struct rk_iovmm *vmm)
-{
- int ret = 0;
-
- vmm->vmm_pool = gen_pool_create(PAGE_SHIFT, -1);
- if (!vmm->vmm_pool) {
- ret = -ENOMEM;
- goto err_setup_genalloc;
- }
-
- /* (1GB - 4KB) addr space from 0x10000000 */
- ret = gen_pool_add(vmm->vmm_pool, IOVA_START, IOVM_SIZE, -1);
- if (ret)
- goto err_setup_domain;
-
- vmm->domain = iommu_domain_alloc(&platform_bus_type);
- if (!vmm->domain) {
- ret = -ENOMEM;
- goto err_setup_domain;
- }
-
- spin_lock_init(&vmm->lock);
-
- INIT_LIST_HEAD(&vmm->regions_list);
-
- dev_info(iommu, "IOVMM: Created %#x B IOVMM from %#x.\n",
- IOVM_SIZE, IOVA_START);
- return 0;
-err_setup_domain:
- gen_pool_destroy(vmm->vmm_pool);
-err_setup_genalloc:
- dev_err(iommu, "IOVMM: Failed to create IOVMM (%d)\n", ret);
-
- return ret;
-}
#define VOP_IOMMU_COMPATIBLE_NAME "rockchip,vop_mmu"
#define HEVC_IOMMU_COMPATIBLE_NAME "rockchip,hevc_mmu"
#define VPU_IOMMU_COMPATIBLE_NAME "rockchip,vpu_mmu"
+#define VDEC_IOMMU_COMPATIBLE_NAME "rockchip,vdec_mmu"
enum rk_iommu_inttype {
IOMMU_PAGEFAULT,
struct scatterlist;
struct device;
-#ifdef CONFIG_ROCKCHIP_IOVMM
+#ifdef CONFIG_RK_IOVMM
int rockchip_iovmm_activate(struct device *dev);
void rockchip_iovmm_deactivate(struct device *dev);
return -ENOSYS;
}
-#endif /* CONFIG_ROCKCHIP_IOVMM */
+#endif /* CONFIG_RK_IOVMM */
#endif /*__ASM_PLAT_IOVMM_H*/
projects / firefly-linux-kernel-4.4.55.git / commitdiff