2 * Copyright (C) 2011-2014 NVIDIA CORPORATION. All rights reserved.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
9 #include <linux/bitops.h>
10 #include <linux/err.h>
11 #include <linux/iommu.h>
12 #include <linux/kernel.h>
14 #include <linux/of_device.h>
15 #include <linux/platform_device.h>
16 #include <linux/slab.h>
18 #include <soc/tegra/ahb.h>
19 #include <soc/tegra/mc.h>
26 const struct tegra_smmu_soc *soc;
28 unsigned long pfn_mask;
33 struct list_head list;
36 struct tegra_smmu_as {
37 struct iommu_domain domain;
38 struct tegra_smmu *smmu;
39 unsigned int use_count;
46 static struct tegra_smmu_as *to_smmu_as(struct iommu_domain *dom)
48 return container_of(dom, struct tegra_smmu_as, domain);
51 static inline void smmu_writel(struct tegra_smmu *smmu, u32 value,
54 writel(value, smmu->regs + offset);
57 static inline u32 smmu_readl(struct tegra_smmu *smmu, unsigned long offset)
59 return readl(smmu->regs + offset);
62 #define SMMU_CONFIG 0x010
63 #define SMMU_CONFIG_ENABLE (1 << 0)
65 #define SMMU_TLB_CONFIG 0x14
66 #define SMMU_TLB_CONFIG_HIT_UNDER_MISS (1 << 29)
67 #define SMMU_TLB_CONFIG_ROUND_ROBIN_ARBITRATION (1 << 28)
68 #define SMMU_TLB_CONFIG_ACTIVE_LINES(x) ((x) & 0x3f)
70 #define SMMU_PTC_CONFIG 0x18
71 #define SMMU_PTC_CONFIG_ENABLE (1 << 29)
72 #define SMMU_PTC_CONFIG_REQ_LIMIT(x) (((x) & 0x0f) << 24)
73 #define SMMU_PTC_CONFIG_INDEX_MAP(x) ((x) & 0x3f)
75 #define SMMU_PTB_ASID 0x01c
76 #define SMMU_PTB_ASID_VALUE(x) ((x) & 0x7f)
78 #define SMMU_PTB_DATA 0x020
79 #define SMMU_PTB_DATA_VALUE(page, attr) (page_to_phys(page) >> 12 | (attr))
81 #define SMMU_MK_PDE(page, attr) (page_to_phys(page) >> SMMU_PTE_SHIFT | (attr))
83 #define SMMU_TLB_FLUSH 0x030
84 #define SMMU_TLB_FLUSH_VA_MATCH_ALL (0 << 0)
85 #define SMMU_TLB_FLUSH_VA_MATCH_SECTION (2 << 0)
86 #define SMMU_TLB_FLUSH_VA_MATCH_GROUP (3 << 0)
87 #define SMMU_TLB_FLUSH_ASID(x) (((x) & 0x7f) << 24)
88 #define SMMU_TLB_FLUSH_VA_SECTION(addr) ((((addr) & 0xffc00000) >> 12) | \
89 SMMU_TLB_FLUSH_VA_MATCH_SECTION)
90 #define SMMU_TLB_FLUSH_VA_GROUP(addr) ((((addr) & 0xffffc000) >> 12) | \
91 SMMU_TLB_FLUSH_VA_MATCH_GROUP)
92 #define SMMU_TLB_FLUSH_ASID_MATCH (1 << 31)
94 #define SMMU_PTC_FLUSH 0x034
95 #define SMMU_PTC_FLUSH_TYPE_ALL (0 << 0)
96 #define SMMU_PTC_FLUSH_TYPE_ADR (1 << 0)
98 #define SMMU_PTC_FLUSH_HI 0x9b8
99 #define SMMU_PTC_FLUSH_HI_MASK 0x3
101 /* per-SWGROUP SMMU_*_ASID register */
102 #define SMMU_ASID_ENABLE (1 << 31)
103 #define SMMU_ASID_MASK 0x7f
104 #define SMMU_ASID_VALUE(x) ((x) & SMMU_ASID_MASK)
106 /* page table definitions */
107 #define SMMU_NUM_PDE 1024
108 #define SMMU_NUM_PTE 1024
110 #define SMMU_SIZE_PD (SMMU_NUM_PDE * 4)
111 #define SMMU_SIZE_PT (SMMU_NUM_PTE * 4)
113 #define SMMU_PDE_SHIFT 22
114 #define SMMU_PTE_SHIFT 12
116 #define SMMU_PD_READABLE (1 << 31)
117 #define SMMU_PD_WRITABLE (1 << 30)
118 #define SMMU_PD_NONSECURE (1 << 29)
120 #define SMMU_PDE_READABLE (1 << 31)
121 #define SMMU_PDE_WRITABLE (1 << 30)
122 #define SMMU_PDE_NONSECURE (1 << 29)
123 #define SMMU_PDE_NEXT (1 << 28)
125 #define SMMU_PTE_READABLE (1 << 31)
126 #define SMMU_PTE_WRITABLE (1 << 30)
127 #define SMMU_PTE_NONSECURE (1 << 29)
129 #define SMMU_PDE_ATTR (SMMU_PDE_READABLE | SMMU_PDE_WRITABLE | \
131 #define SMMU_PTE_ATTR (SMMU_PTE_READABLE | SMMU_PTE_WRITABLE | \
134 static inline void smmu_flush_ptc(struct tegra_smmu *smmu, struct page *page,
135 unsigned long offset)
137 phys_addr_t phys = page ? page_to_phys(page) : 0;
141 offset &= ~(smmu->mc->soc->atom_size - 1);
143 if (smmu->mc->soc->num_address_bits > 32) {
144 #ifdef CONFIG_PHYS_ADDR_T_64BIT
145 value = (phys >> 32) & SMMU_PTC_FLUSH_HI_MASK;
149 smmu_writel(smmu, value, SMMU_PTC_FLUSH_HI);
152 value = (phys + offset) | SMMU_PTC_FLUSH_TYPE_ADR;
154 value = SMMU_PTC_FLUSH_TYPE_ALL;
157 smmu_writel(smmu, value, SMMU_PTC_FLUSH);
160 static inline void smmu_flush_tlb(struct tegra_smmu *smmu)
162 smmu_writel(smmu, SMMU_TLB_FLUSH_VA_MATCH_ALL, SMMU_TLB_FLUSH);
165 static inline void smmu_flush_tlb_asid(struct tegra_smmu *smmu,
170 value = SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_ASID(asid) |
171 SMMU_TLB_FLUSH_VA_MATCH_ALL;
172 smmu_writel(smmu, value, SMMU_TLB_FLUSH);
175 static inline void smmu_flush_tlb_section(struct tegra_smmu *smmu,
181 value = SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_ASID(asid) |
182 SMMU_TLB_FLUSH_VA_SECTION(iova);
183 smmu_writel(smmu, value, SMMU_TLB_FLUSH);
186 static inline void smmu_flush_tlb_group(struct tegra_smmu *smmu,
192 value = SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_ASID(asid) |
193 SMMU_TLB_FLUSH_VA_GROUP(iova);
194 smmu_writel(smmu, value, SMMU_TLB_FLUSH);
197 static inline void smmu_flush(struct tegra_smmu *smmu)
199 smmu_readl(smmu, SMMU_CONFIG);
202 static int tegra_smmu_alloc_asid(struct tegra_smmu *smmu, unsigned int *idp)
206 mutex_lock(&smmu->lock);
208 id = find_first_zero_bit(smmu->asids, smmu->soc->num_asids);
209 if (id >= smmu->soc->num_asids) {
210 mutex_unlock(&smmu->lock);
214 set_bit(id, smmu->asids);
217 mutex_unlock(&smmu->lock);
221 static void tegra_smmu_free_asid(struct tegra_smmu *smmu, unsigned int id)
223 mutex_lock(&smmu->lock);
224 clear_bit(id, smmu->asids);
225 mutex_unlock(&smmu->lock);
228 static bool tegra_smmu_capable(enum iommu_cap cap)
233 static struct iommu_domain *tegra_smmu_domain_alloc(unsigned type)
235 struct tegra_smmu_as *as;
239 if (type != IOMMU_DOMAIN_UNMANAGED)
242 as = kzalloc(sizeof(*as), GFP_KERNEL);
246 as->attr = SMMU_PD_READABLE | SMMU_PD_WRITABLE | SMMU_PD_NONSECURE;
248 as->pd = alloc_page(GFP_KERNEL | __GFP_DMA);
254 as->count = alloc_page(GFP_KERNEL);
262 pd = page_address(as->pd);
263 SetPageReserved(as->pd);
265 for (i = 0; i < SMMU_NUM_PDE; i++)
268 /* clear PDE usage counters */
269 pd = page_address(as->count);
270 SetPageReserved(as->count);
272 for (i = 0; i < SMMU_NUM_PDE; i++)
276 as->domain.geometry.aperture_start = 0;
277 as->domain.geometry.aperture_end = 0xffffffff;
278 as->domain.geometry.force_aperture = true;
283 static void tegra_smmu_domain_free(struct iommu_domain *domain)
285 struct tegra_smmu_as *as = to_smmu_as(domain);
287 /* TODO: free page directory and page tables */
288 ClearPageReserved(as->pd);
293 static const struct tegra_smmu_swgroup *
294 tegra_smmu_find_swgroup(struct tegra_smmu *smmu, unsigned int swgroup)
296 const struct tegra_smmu_swgroup *group = NULL;
299 for (i = 0; i < smmu->soc->num_swgroups; i++) {
300 if (smmu->soc->swgroups[i].swgroup == swgroup) {
301 group = &smmu->soc->swgroups[i];
309 static void tegra_smmu_enable(struct tegra_smmu *smmu, unsigned int swgroup,
312 const struct tegra_smmu_swgroup *group;
316 for (i = 0; i < smmu->soc->num_clients; i++) {
317 const struct tegra_mc_client *client = &smmu->soc->clients[i];
319 if (client->swgroup != swgroup)
322 value = smmu_readl(smmu, client->smmu.reg);
323 value |= BIT(client->smmu.bit);
324 smmu_writel(smmu, value, client->smmu.reg);
327 group = tegra_smmu_find_swgroup(smmu, swgroup);
329 value = smmu_readl(smmu, group->reg);
330 value &= ~SMMU_ASID_MASK;
331 value |= SMMU_ASID_VALUE(asid);
332 value |= SMMU_ASID_ENABLE;
333 smmu_writel(smmu, value, group->reg);
337 static void tegra_smmu_disable(struct tegra_smmu *smmu, unsigned int swgroup,
340 const struct tegra_smmu_swgroup *group;
344 group = tegra_smmu_find_swgroup(smmu, swgroup);
346 value = smmu_readl(smmu, group->reg);
347 value &= ~SMMU_ASID_MASK;
348 value |= SMMU_ASID_VALUE(asid);
349 value &= ~SMMU_ASID_ENABLE;
350 smmu_writel(smmu, value, group->reg);
353 for (i = 0; i < smmu->soc->num_clients; i++) {
354 const struct tegra_mc_client *client = &smmu->soc->clients[i];
356 if (client->swgroup != swgroup)
359 value = smmu_readl(smmu, client->smmu.reg);
360 value &= ~BIT(client->smmu.bit);
361 smmu_writel(smmu, value, client->smmu.reg);
365 static int tegra_smmu_as_prepare(struct tegra_smmu *smmu,
366 struct tegra_smmu_as *as)
371 if (as->use_count > 0) {
376 err = tegra_smmu_alloc_asid(smmu, &as->id);
380 smmu->soc->ops->flush_dcache(as->pd, 0, SMMU_SIZE_PD);
381 smmu_flush_ptc(smmu, as->pd, 0);
382 smmu_flush_tlb_asid(smmu, as->id);
384 smmu_writel(smmu, as->id & 0x7f, SMMU_PTB_ASID);
385 value = SMMU_PTB_DATA_VALUE(as->pd, as->attr);
386 smmu_writel(smmu, value, SMMU_PTB_DATA);
395 static void tegra_smmu_as_unprepare(struct tegra_smmu *smmu,
396 struct tegra_smmu_as *as)
398 if (--as->use_count > 0)
401 tegra_smmu_free_asid(smmu, as->id);
405 static int tegra_smmu_attach_dev(struct iommu_domain *domain,
408 struct tegra_smmu *smmu = dev->archdata.iommu;
409 struct tegra_smmu_as *as = to_smmu_as(domain);
410 struct device_node *np = dev->of_node;
411 struct of_phandle_args args;
412 unsigned int index = 0;
415 while (!of_parse_phandle_with_args(np, "iommus", "#iommu-cells", index,
417 unsigned int swgroup = args.args[0];
419 if (args.np != smmu->dev->of_node) {
420 of_node_put(args.np);
424 of_node_put(args.np);
426 err = tegra_smmu_as_prepare(smmu, as);
430 tegra_smmu_enable(smmu, swgroup, as->id);
440 static void tegra_smmu_detach_dev(struct iommu_domain *domain, struct device *dev)
442 struct tegra_smmu_as *as = to_smmu_as(domain);
443 struct device_node *np = dev->of_node;
444 struct tegra_smmu *smmu = as->smmu;
445 struct of_phandle_args args;
446 unsigned int index = 0;
448 while (!of_parse_phandle_with_args(np, "iommus", "#iommu-cells", index,
450 unsigned int swgroup = args.args[0];
452 if (args.np != smmu->dev->of_node) {
453 of_node_put(args.np);
457 of_node_put(args.np);
459 tegra_smmu_disable(smmu, swgroup, as->id);
460 tegra_smmu_as_unprepare(smmu, as);
465 static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova,
468 u32 *pd = page_address(as->pd), *pt, *count;
469 u32 pde = (iova >> SMMU_PDE_SHIFT) & 0x3ff;
470 u32 pte = (iova >> SMMU_PTE_SHIFT) & 0x3ff;
471 struct tegra_smmu *smmu = as->smmu;
476 page = alloc_page(GFP_KERNEL | __GFP_DMA);
480 pt = page_address(page);
481 SetPageReserved(page);
483 for (i = 0; i < SMMU_NUM_PTE; i++)
486 smmu->soc->ops->flush_dcache(page, 0, SMMU_SIZE_PT);
488 pd[pde] = SMMU_MK_PDE(page, SMMU_PDE_ATTR | SMMU_PDE_NEXT);
490 smmu->soc->ops->flush_dcache(as->pd, pde << 2, 4);
491 smmu_flush_ptc(smmu, as->pd, pde << 2);
492 smmu_flush_tlb_section(smmu, as->id, iova);
495 page = pfn_to_page(pd[pde] & smmu->pfn_mask);
496 pt = page_address(page);
501 /* Keep track of entries in this page table. */
502 count = page_address(as->count);
509 static void as_put_pte(struct tegra_smmu_as *as, dma_addr_t iova)
511 u32 pde = (iova >> SMMU_PDE_SHIFT) & 0x3ff;
512 u32 pte = (iova >> SMMU_PTE_SHIFT) & 0x3ff;
513 u32 *count = page_address(as->count);
514 u32 *pd = page_address(as->pd), *pt;
517 page = pfn_to_page(pd[pde] & as->smmu->pfn_mask);
518 pt = page_address(page);
521 * When no entries in this page table are used anymore, return the
522 * memory page to the system.
525 if (--count[pde] == 0) {
526 ClearPageReserved(page);
535 static int tegra_smmu_map(struct iommu_domain *domain, unsigned long iova,
536 phys_addr_t paddr, size_t size, int prot)
538 struct tegra_smmu_as *as = to_smmu_as(domain);
539 struct tegra_smmu *smmu = as->smmu;
540 unsigned long offset;
544 pte = as_get_pte(as, iova, &page);
548 *pte = __phys_to_pfn(paddr) | SMMU_PTE_ATTR;
549 offset = offset_in_page(pte);
551 smmu->soc->ops->flush_dcache(page, offset, 4);
552 smmu_flush_ptc(smmu, page, offset);
553 smmu_flush_tlb_group(smmu, as->id, iova);
559 static size_t tegra_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
562 struct tegra_smmu_as *as = to_smmu_as(domain);
563 struct tegra_smmu *smmu = as->smmu;
564 unsigned long offset;
568 pte = as_get_pte(as, iova, &page);
572 offset = offset_in_page(pte);
573 as_put_pte(as, iova);
575 smmu->soc->ops->flush_dcache(page, offset, 4);
576 smmu_flush_ptc(smmu, page, offset);
577 smmu_flush_tlb_group(smmu, as->id, iova);
583 static phys_addr_t tegra_smmu_iova_to_phys(struct iommu_domain *domain,
586 struct tegra_smmu_as *as = to_smmu_as(domain);
591 pte = as_get_pte(as, iova, &page);
592 pfn = *pte & as->smmu->pfn_mask;
594 return PFN_PHYS(pfn);
597 static struct tegra_smmu *tegra_smmu_find(struct device_node *np)
599 struct platform_device *pdev;
602 pdev = of_find_device_by_node(np);
606 mc = platform_get_drvdata(pdev);
613 static int tegra_smmu_add_device(struct device *dev)
615 struct device_node *np = dev->of_node;
616 struct of_phandle_args args;
617 unsigned int index = 0;
619 while (of_parse_phandle_with_args(np, "iommus", "#iommu-cells", index,
621 struct tegra_smmu *smmu;
623 smmu = tegra_smmu_find(args.np);
626 * Only a single IOMMU master interface is currently
627 * supported by the Linux kernel, so abort after the
630 dev->archdata.iommu = smmu;
640 static void tegra_smmu_remove_device(struct device *dev)
642 dev->archdata.iommu = NULL;
645 static const struct iommu_ops tegra_smmu_ops = {
646 .capable = tegra_smmu_capable,
647 .domain_alloc = tegra_smmu_domain_alloc,
648 .domain_free = tegra_smmu_domain_free,
649 .attach_dev = tegra_smmu_attach_dev,
650 .detach_dev = tegra_smmu_detach_dev,
651 .add_device = tegra_smmu_add_device,
652 .remove_device = tegra_smmu_remove_device,
653 .map = tegra_smmu_map,
654 .unmap = tegra_smmu_unmap,
655 .map_sg = default_iommu_map_sg,
656 .iova_to_phys = tegra_smmu_iova_to_phys,
658 .pgsize_bitmap = SZ_4K,
661 static void tegra_smmu_ahb_enable(void)
663 static const struct of_device_id ahb_match[] = {
664 { .compatible = "nvidia,tegra30-ahb", },
667 struct device_node *ahb;
669 ahb = of_find_matching_node(NULL, ahb_match);
671 tegra_ahb_enable_smmu(ahb);
676 struct tegra_smmu *tegra_smmu_probe(struct device *dev,
677 const struct tegra_smmu_soc *soc,
680 struct tegra_smmu *smmu;
685 /* This can happen on Tegra20 which doesn't have an SMMU */
689 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
691 return ERR_PTR(-ENOMEM);
694 * This is a bit of a hack. Ideally we'd want to simply return this
695 * value. However the IOMMU registration process will attempt to add
696 * all devices to the IOMMU when bus_set_iommu() is called. In order
697 * not to rely on global variables to track the IOMMU instance, we
698 * set it here so that it can be looked up from the .add_device()
699 * callback via the IOMMU device's .drvdata field.
703 size = BITS_TO_LONGS(soc->num_asids) * sizeof(long);
705 smmu->asids = devm_kzalloc(dev, size, GFP_KERNEL);
707 return ERR_PTR(-ENOMEM);
709 mutex_init(&smmu->lock);
711 smmu->regs = mc->regs;
716 smmu->pfn_mask = BIT_MASK(mc->soc->num_address_bits - PAGE_SHIFT) - 1;
717 dev_dbg(dev, "address bits: %u, PFN mask: %#lx\n",
718 mc->soc->num_address_bits, smmu->pfn_mask);
720 value = SMMU_PTC_CONFIG_ENABLE | SMMU_PTC_CONFIG_INDEX_MAP(0x3f);
722 if (soc->supports_request_limit)
723 value |= SMMU_PTC_CONFIG_REQ_LIMIT(8);
725 smmu_writel(smmu, value, SMMU_PTC_CONFIG);
727 value = SMMU_TLB_CONFIG_HIT_UNDER_MISS |
728 SMMU_TLB_CONFIG_ACTIVE_LINES(0x20);
730 if (soc->supports_round_robin_arbitration)
731 value |= SMMU_TLB_CONFIG_ROUND_ROBIN_ARBITRATION;
733 smmu_writel(smmu, value, SMMU_TLB_CONFIG);
735 smmu_flush_ptc(smmu, NULL, 0);
736 smmu_flush_tlb(smmu);
737 smmu_writel(smmu, SMMU_CONFIG_ENABLE, SMMU_CONFIG);
740 tegra_smmu_ahb_enable();
742 err = bus_set_iommu(&platform_bus_type, &tegra_smmu_ops);