drivers/iommu/exynos-iommu.c

   1 /* linux/drivers/iommu/exynos_iommu.c
   2  *
   3  * Copyright (c) 2011 Samsung Electronics Co., Ltd.
   4  *              http://www.samsung.com
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 as
   8  * published by the Free Software Foundation.
   9  */
  10
  11 #ifdef CONFIG_EXYNOS_IOMMU_DEBUG
  12 #define DEBUG
  13 #endif
  14
  15 #include <linux/clk.h>
  16 #include <linux/err.h>
  17 #include <linux/io.h>
  18 #include <linux/iommu.h>
  19 #include <linux/interrupt.h>
  20 #include <linux/list.h>
  21 #include <linux/platform_device.h>
  22 #include <linux/pm_runtime.h>
  23 #include <linux/slab.h>
  24
  25 #include <asm/cacheflush.h>
  26 #include <asm/pgtable.h>
  27
  28 typedef u32 sysmmu_iova_t;
  29 typedef u32 sysmmu_pte_t;
  30
  31 /* We do not consider super section mapping (16MB) */
  32 #define SECT_ORDER 20
  33 #define LPAGE_ORDER 16
  34 #define SPAGE_ORDER 12
  35
  36 #define SECT_SIZE (1 << SECT_ORDER)
  37 #define LPAGE_SIZE (1 << LPAGE_ORDER)
  38 #define SPAGE_SIZE (1 << SPAGE_ORDER)
  39
  40 #define SECT_MASK (~(SECT_SIZE - 1))
  41 #define LPAGE_MASK (~(LPAGE_SIZE - 1))
  42 #define SPAGE_MASK (~(SPAGE_SIZE - 1))
  43
  44 #define lv1ent_fault(sent) ((*(sent) == ZERO_LV2LINK) || \
  45                            ((*(sent) & 3) == 0) || ((*(sent) & 3) == 3))
  46 #define lv1ent_zero(sent) (*(sent) == ZERO_LV2LINK)
  47 #define lv1ent_page_zero(sent) ((*(sent) & 3) == 1)
  48 #define lv1ent_page(sent) ((*(sent) != ZERO_LV2LINK) && \
  49                           ((*(sent) & 3) == 1))
  50 #define lv1ent_section(sent) ((*(sent) & 3) == 2)
  51
  52 #define lv2ent_fault(pent) ((*(pent) & 3) == 0)
  53 #define lv2ent_small(pent) ((*(pent) & 2) == 2)
  54 #define lv2ent_large(pent) ((*(pent) & 3) == 1)
  55
  56 static u32 sysmmu_page_offset(sysmmu_iova_t iova, u32 size)
  57 {
  58         return iova & (size - 1);
  59 }
  60
  61 #define section_phys(sent) (*(sent) & SECT_MASK)
  62 #define section_offs(iova) sysmmu_page_offset((iova), SECT_SIZE)
  63 #define lpage_phys(pent) (*(pent) & LPAGE_MASK)
  64 #define lpage_offs(iova) sysmmu_page_offset((iova), LPAGE_SIZE)
  65 #define spage_phys(pent) (*(pent) & SPAGE_MASK)
  66 #define spage_offs(iova) sysmmu_page_offset((iova), SPAGE_SIZE)
  67
  68 #define NUM_LV1ENTRIES 4096
  69 #define NUM_LV2ENTRIES (SECT_SIZE / SPAGE_SIZE)
  70
  71 static u32 lv1ent_offset(sysmmu_iova_t iova)
  72 {
  73         return iova >> SECT_ORDER;
  74 }
  75
  76 static u32 lv2ent_offset(sysmmu_iova_t iova)
  77 {
  78         return (iova >> SPAGE_ORDER) & (NUM_LV2ENTRIES - 1);
  79 }
  80
  81 #define LV2TABLE_SIZE (NUM_LV2ENTRIES * sizeof(sysmmu_pte_t))
  82
  83 #define SPAGES_PER_LPAGE (LPAGE_SIZE / SPAGE_SIZE)
  84
  85 #define lv2table_base(sent) (*(sent) & 0xFFFFFC00)
  86
  87 #define mk_lv1ent_sect(pa) ((pa) | 2)
  88 #define mk_lv1ent_page(pa) ((pa) | 1)
  89 #define mk_lv2ent_lpage(pa) ((pa) | 1)
  90 #define mk_lv2ent_spage(pa) ((pa) | 2)
  91
  92 #define CTRL_ENABLE     0x5
  93 #define CTRL_BLOCK      0x7
  94 #define CTRL_DISABLE    0x0
  95
  96 #define CFG_LRU         0x1
  97 #define CFG_QOS(n)      ((n & 0xF) << 7)
  98 #define CFG_MASK        0x0150FFFF /* Selecting bit 0-15, 20, 22 and 24 */
  99 #define CFG_ACGEN       (1 << 24) /* System MMU 3.3 only */
 100 #define CFG_SYSSEL      (1 << 22) /* System MMU 3.2 only */
 101 #define CFG_FLPDCACHE   (1 << 20) /* System MMU 3.2+ only */
 102
 103 #define REG_MMU_CTRL            0x000
 104 #define REG_MMU_CFG             0x004
 105 #define REG_MMU_STATUS          0x008
 106 #define REG_MMU_FLUSH           0x00C
 107 #define REG_MMU_FLUSH_ENTRY     0x010
 108 #define REG_PT_BASE_ADDR        0x014
 109 #define REG_INT_STATUS          0x018
 110 #define REG_INT_CLEAR           0x01C
 111
 112 #define REG_PAGE_FAULT_ADDR     0x024
 113 #define REG_AW_FAULT_ADDR       0x028
 114 #define REG_AR_FAULT_ADDR       0x02C
 115 #define REG_DEFAULT_SLAVE_ADDR  0x030
 116
 117 #define REG_MMU_VERSION         0x034
 118
 119 #define MMU_MAJ_VER(val)        ((val) >> 7)
 120 #define MMU_MIN_VER(val)        ((val) & 0x7F)
 121 #define MMU_RAW_VER(reg)        (((reg) >> 21) & ((1 << 11) - 1)) /* 11 bits */
 122
 123 #define MAKE_MMU_VER(maj, min)  ((((maj) & 0xF) << 7) | ((min) & 0x7F))
 124
 125 #define REG_PB0_SADDR           0x04C
 126 #define REG_PB0_EADDR           0x050
 127 #define REG_PB1_SADDR           0x054
 128 #define REG_PB1_EADDR           0x058
 129
 130 #define has_sysmmu(dev)         (dev->archdata.iommu != NULL)
 131
 132 static struct kmem_cache *lv2table_kmem_cache;
 133 static sysmmu_pte_t *zero_lv2_table;
 134 #define ZERO_LV2LINK mk_lv1ent_page(virt_to_phys(zero_lv2_table))
 135
 136 static sysmmu_pte_t *section_entry(sysmmu_pte_t *pgtable, sysmmu_iova_t iova)
 137 {
 138         return pgtable + lv1ent_offset(iova);
 139 }
 140
 141 static sysmmu_pte_t *page_entry(sysmmu_pte_t *sent, sysmmu_iova_t iova)
 142 {
 143         return (sysmmu_pte_t *)phys_to_virt(
 144                                 lv2table_base(sent)) + lv2ent_offset(iova);
 145 }
 146
 147 enum exynos_sysmmu_inttype {
 148         SYSMMU_PAGEFAULT,
 149         SYSMMU_AR_MULTIHIT,
 150         SYSMMU_AW_MULTIHIT,
 151         SYSMMU_BUSERROR,
 152         SYSMMU_AR_SECURITY,
 153         SYSMMU_AR_ACCESS,
 154         SYSMMU_AW_SECURITY,
 155         SYSMMU_AW_PROTECTION, /* 7 */
 156         SYSMMU_FAULT_UNKNOWN,
 157         SYSMMU_FAULTS_NUM
 158 };
 159
 160 static unsigned short fault_reg_offset[SYSMMU_FAULTS_NUM] = {
 161         REG_PAGE_FAULT_ADDR,
 162         REG_AR_FAULT_ADDR,
 163         REG_AW_FAULT_ADDR,
 164         REG_DEFAULT_SLAVE_ADDR,
 165         REG_AR_FAULT_ADDR,
 166         REG_AR_FAULT_ADDR,
 167         REG_AW_FAULT_ADDR,
 168         REG_AW_FAULT_ADDR
 169 };
 170
 171 static char *sysmmu_fault_name[SYSMMU_FAULTS_NUM] = {
 172         "PAGE FAULT",
 173         "AR MULTI-HIT FAULT",
 174         "AW MULTI-HIT FAULT",
 175         "BUS ERROR",
 176         "AR SECURITY PROTECTION FAULT",
 177         "AR ACCESS PROTECTION FAULT",
 178         "AW SECURITY PROTECTION FAULT",
 179         "AW ACCESS PROTECTION FAULT",
 180         "UNKNOWN FAULT"
 181 };
 182
 183 /* attached to dev.archdata.iommu of the master device */
 184 struct exynos_iommu_owner {
 185         struct device *sysmmu;
 186 };
 187
 188 struct exynos_iommu_domain {
 189         struct list_head clients; /* list of sysmmu_drvdata.node */
 190         sysmmu_pte_t *pgtable; /* lv1 page table, 16KB */
 191         short *lv2entcnt; /* free lv2 entry counter for each section */
 192         spinlock_t lock; /* lock for this structure */
 193         spinlock_t pgtablelock; /* lock for modifying page table @ pgtable */
 194         struct iommu_domain domain; /* generic domain data structure */
 195 };
 196
 197 struct sysmmu_drvdata {
 198         struct device *sysmmu;  /* System MMU's device descriptor */
 199         struct device *master;  /* Owner of system MMU */
 200         void __iomem *sfrbase;
 201         struct clk *clk;
 202         struct clk *clk_master;
 203         int activations;
 204         spinlock_t lock;
 205         struct exynos_iommu_domain *domain;
 206         struct list_head domain_node;
 207         phys_addr_t pgtable;
 208         unsigned int version;
 209 };
 210
 211 static struct exynos_iommu_domain *to_exynos_domain(struct iommu_domain *dom)
 212 {
 213         return container_of(dom, struct exynos_iommu_domain, domain);
 214 }
 215
 216 static bool set_sysmmu_active(struct sysmmu_drvdata *data)
 217 {
 218         /* return true if the System MMU was not active previously
 219            and it needs to be initialized */
 220         return ++data->activations == 1;
 221 }
 222
 223 static bool set_sysmmu_inactive(struct sysmmu_drvdata *data)
 224 {
 225         /* return true if the System MMU is needed to be disabled */
 226         BUG_ON(data->activations < 1);
 227         return --data->activations == 0;
 228 }
 229
 230 static bool is_sysmmu_active(struct sysmmu_drvdata *data)
 231 {
 232         return data->activations > 0;
 233 }
 234
 235 static void sysmmu_unblock(void __iomem *sfrbase)
 236 {
 237         __raw_writel(CTRL_ENABLE, sfrbase + REG_MMU_CTRL);
 238 }
 239
 240 static bool sysmmu_block(void __iomem *sfrbase)
 241 {
 242         int i = 120;
 243
 244         __raw_writel(CTRL_BLOCK, sfrbase + REG_MMU_CTRL);
 245         while ((i > 0) && !(__raw_readl(sfrbase + REG_MMU_STATUS) & 1))
 246                 --i;
 247
 248         if (!(__raw_readl(sfrbase + REG_MMU_STATUS) & 1)) {
 249                 sysmmu_unblock(sfrbase);
 250                 return false;
 251         }
 252
 253         return true;
 254 }
 255
 256 static void __sysmmu_tlb_invalidate(void __iomem *sfrbase)
 257 {
 258         __raw_writel(0x1, sfrbase + REG_MMU_FLUSH);
 259 }
 260
 261 static void __sysmmu_tlb_invalidate_entry(void __iomem *sfrbase,
 262                                 sysmmu_iova_t iova, unsigned int num_inv)
 263 {
 264         unsigned int i;
 265
 266         for (i = 0; i < num_inv; i++) {
 267                 __raw_writel((iova & SPAGE_MASK) | 1,
 268                                 sfrbase + REG_MMU_FLUSH_ENTRY);
 269                 iova += SPAGE_SIZE;
 270         }
 271 }
 272
 273 static void __sysmmu_set_ptbase(void __iomem *sfrbase,
 274                                        phys_addr_t pgd)
 275 {
 276         __raw_writel(pgd, sfrbase + REG_PT_BASE_ADDR);
 277
 278         __sysmmu_tlb_invalidate(sfrbase);
 279 }
 280
 281 static void show_fault_information(const char *name,
 282                 enum exynos_sysmmu_inttype itype,
 283                 phys_addr_t pgtable_base, sysmmu_iova_t fault_addr)
 284 {
 285         sysmmu_pte_t *ent;
 286
 287         if ((itype >= SYSMMU_FAULTS_NUM) || (itype < SYSMMU_PAGEFAULT))
 288                 itype = SYSMMU_FAULT_UNKNOWN;
 289
 290         pr_err("%s occurred at %#x by %s(Page table base: %pa)\n",
 291                 sysmmu_fault_name[itype], fault_addr, name, &pgtable_base);
 292
 293         ent = section_entry(phys_to_virt(pgtable_base), fault_addr);
 294         pr_err("\tLv1 entry: %#x\n", *ent);
 295
 296         if (lv1ent_page(ent)) {
 297                 ent = page_entry(ent, fault_addr);
 298                 pr_err("\t Lv2 entry: %#x\n", *ent);
 299         }
 300 }
 301
 302 static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id)
 303 {
 304         /* SYSMMU is in blocked state when interrupt occurred. */
 305         struct sysmmu_drvdata *data = dev_id;
 306         enum exynos_sysmmu_inttype itype;
 307         sysmmu_iova_t addr = -1;
 308         int ret = -ENOSYS;
 309
 310         WARN_ON(!is_sysmmu_active(data));
 311
 312         spin_lock(&data->lock);
 313
 314         if (!IS_ERR(data->clk_master))
 315                 clk_enable(data->clk_master);
 316
 317         itype = (enum exynos_sysmmu_inttype)
 318                 __ffs(__raw_readl(data->sfrbase + REG_INT_STATUS));
 319         if (WARN_ON(!((itype >= 0) && (itype < SYSMMU_FAULT_UNKNOWN))))
 320                 itype = SYSMMU_FAULT_UNKNOWN;
 321         else
 322                 addr = __raw_readl(data->sfrbase + fault_reg_offset[itype]);
 323
 324         if (itype == SYSMMU_FAULT_UNKNOWN) {
 325                 pr_err("%s: Fault is not occurred by System MMU '%s'!\n",
 326                         __func__, dev_name(data->sysmmu));
 327                 pr_err("%s: Please check if IRQ is correctly configured.\n",
 328                         __func__);
 329                 BUG();
 330         } else {
 331                 unsigned int base =
 332                                 __raw_readl(data->sfrbase + REG_PT_BASE_ADDR);
 333                 show_fault_information(dev_name(data->sysmmu),
 334                                         itype, base, addr);
 335                 if (data->domain)
 336                         ret = report_iommu_fault(&data->domain->domain,
 337                                         data->master, addr, itype);
 338         }
 339
 340         /* fault is not recovered by fault handler */
 341         BUG_ON(ret != 0);
 342
 343         __raw_writel(1 << itype, data->sfrbase + REG_INT_CLEAR);
 344
 345         sysmmu_unblock(data->sfrbase);
 346
 347         if (!IS_ERR(data->clk_master))
 348                 clk_disable(data->clk_master);
 349
 350         spin_unlock(&data->lock);
 351
 352         return IRQ_HANDLED;
 353 }
 354
 355 static void __sysmmu_disable_nocount(struct sysmmu_drvdata *data)
 356 {
 357         if (!IS_ERR(data->clk_master))
 358                 clk_enable(data->clk_master);
 359
 360         __raw_writel(CTRL_DISABLE, data->sfrbase + REG_MMU_CTRL);
 361         __raw_writel(0, data->sfrbase + REG_MMU_CFG);
 362
 363         clk_disable(data->clk);
 364         if (!IS_ERR(data->clk_master))
 365                 clk_disable(data->clk_master);
 366 }
 367
 368 static bool __sysmmu_disable(struct sysmmu_drvdata *data)
 369 {
 370         bool disabled;
 371         unsigned long flags;
 372
 373         spin_lock_irqsave(&data->lock, flags);
 374
 375         disabled = set_sysmmu_inactive(data);
 376
 377         if (disabled) {
 378                 data->pgtable = 0;
 379                 data->domain = NULL;
 380
 381                 __sysmmu_disable_nocount(data);
 382
 383                 dev_dbg(data->sysmmu, "Disabled\n");
 384         } else  {
 385                 dev_dbg(data->sysmmu, "%d times left to disable\n",
 386                                         data->activations);
 387         }
 388
 389         spin_unlock_irqrestore(&data->lock, flags);
 390
 391         return disabled;
 392 }
 393
 394 static void __sysmmu_init_config(struct sysmmu_drvdata *data)
 395 {
 396         unsigned int cfg = CFG_LRU | CFG_QOS(15);
 397         unsigned int ver;
 398
 399         ver = MMU_RAW_VER(__raw_readl(data->sfrbase + REG_MMU_VERSION));
 400         if (MMU_MAJ_VER(ver) == 3) {
 401                 if (MMU_MIN_VER(ver) >= 2) {
 402                         cfg |= CFG_FLPDCACHE;
 403                         if (MMU_MIN_VER(ver) == 3) {
 404                                 cfg |= CFG_ACGEN;
 405                                 cfg &= ~CFG_LRU;
 406                         } else {
 407                                 cfg |= CFG_SYSSEL;
 408                         }
 409                 }
 410         }
 411
 412         __raw_writel(cfg, data->sfrbase + REG_MMU_CFG);
 413         data->version = ver;
 414 }
 415
 416 static void __sysmmu_enable_nocount(struct sysmmu_drvdata *data)
 417 {
 418         if (!IS_ERR(data->clk_master))
 419                 clk_enable(data->clk_master);
 420         clk_enable(data->clk);
 421
 422         __raw_writel(CTRL_BLOCK, data->sfrbase + REG_MMU_CTRL);
 423
 424         __sysmmu_init_config(data);
 425
 426         __sysmmu_set_ptbase(data->sfrbase, data->pgtable);
 427
 428         __raw_writel(CTRL_ENABLE, data->sfrbase + REG_MMU_CTRL);
 429
 430         if (!IS_ERR(data->clk_master))
 431                 clk_disable(data->clk_master);
 432 }
 433
 434 static int __sysmmu_enable(struct sysmmu_drvdata *data, phys_addr_t pgtable,
 435                            struct exynos_iommu_domain *domain)
 436 {
 437         int ret = 0;
 438         unsigned long flags;
 439
 440         spin_lock_irqsave(&data->lock, flags);
 441         if (set_sysmmu_active(data)) {
 442                 data->pgtable = pgtable;
 443                 data->domain = domain;
 444
 445                 __sysmmu_enable_nocount(data);
 446
 447                 dev_dbg(data->sysmmu, "Enabled\n");
 448         } else {
 449                 ret = (pgtable == data->pgtable) ? 1 : -EBUSY;
 450
 451                 dev_dbg(data->sysmmu, "already enabled\n");
 452         }
 453
 454         if (WARN_ON(ret < 0))
 455                 set_sysmmu_inactive(data); /* decrement count */
 456
 457         spin_unlock_irqrestore(&data->lock, flags);
 458
 459         return ret;
 460 }
 461
 462 static void __sysmmu_tlb_invalidate_flpdcache(struct sysmmu_drvdata *data,
 463                                               sysmmu_iova_t iova)
 464 {
 465         if (data->version == MAKE_MMU_VER(3, 3))
 466                 __raw_writel(iova | 0x1, data->sfrbase + REG_MMU_FLUSH_ENTRY);
 467 }
 468
 469 static void sysmmu_tlb_invalidate_flpdcache(struct sysmmu_drvdata *data,
 470                                             sysmmu_iova_t iova)
 471 {
 472         unsigned long flags;
 473
 474         if (!IS_ERR(data->clk_master))
 475                 clk_enable(data->clk_master);
 476
 477         spin_lock_irqsave(&data->lock, flags);
 478         if (is_sysmmu_active(data))
 479                 __sysmmu_tlb_invalidate_flpdcache(data, iova);
 480         spin_unlock_irqrestore(&data->lock, flags);
 481
 482         if (!IS_ERR(data->clk_master))
 483                 clk_disable(data->clk_master);
 484 }
 485
 486 static void sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data,
 487                                         sysmmu_iova_t iova, size_t size)
 488 {
 489         unsigned long flags;
 490
 491         spin_lock_irqsave(&data->lock, flags);
 492         if (is_sysmmu_active(data)) {
 493                 unsigned int num_inv = 1;
 494
 495                 if (!IS_ERR(data->clk_master))
 496                         clk_enable(data->clk_master);
 497
 498                 /*
 499                  * L2TLB invalidation required
 500                  * 4KB page: 1 invalidation
 501                  * 64KB page: 16 invalidations
 502                  * 1MB page: 64 invalidations
 503                  * because it is set-associative TLB
 504                  * with 8-way and 64 sets.
 505                  * 1MB page can be cached in one of all sets.
 506                  * 64KB page can be one of 16 consecutive sets.
 507                  */
 508                 if (MMU_MAJ_VER(data->version) == 2)
 509                         num_inv = min_t(unsigned int, size / PAGE_SIZE, 64);
 510
 511                 if (sysmmu_block(data->sfrbase)) {
 512                         __sysmmu_tlb_invalidate_entry(
 513                                 data->sfrbase, iova, num_inv);
 514                         sysmmu_unblock(data->sfrbase);
 515                 }
 516                 if (!IS_ERR(data->clk_master))
 517                         clk_disable(data->clk_master);
 518         } else {
 519                 dev_dbg(data->master,
 520                         "disabled. Skipping TLB invalidation @ %#x\n", iova);
 521         }
 522         spin_unlock_irqrestore(&data->lock, flags);
 523 }
 524
 525 static int __init exynos_sysmmu_probe(struct platform_device *pdev)
 526 {
 527         int irq, ret;
 528         struct device *dev = &pdev->dev;
 529         struct sysmmu_drvdata *data;
 530         struct resource *res;
 531
 532         data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
 533         if (!data)
 534                 return -ENOMEM;
 535
 536         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 537         data->sfrbase = devm_ioremap_resource(dev, res);
 538         if (IS_ERR(data->sfrbase))
 539                 return PTR_ERR(data->sfrbase);
 540
 541         irq = platform_get_irq(pdev, 0);
 542         if (irq <= 0) {
 543                 dev_err(dev, "Unable to find IRQ resource\n");
 544                 return irq;
 545         }
 546
 547         ret = devm_request_irq(dev, irq, exynos_sysmmu_irq, 0,
 548                                 dev_name(dev), data);
 549         if (ret) {
 550                 dev_err(dev, "Unabled to register handler of irq %d\n", irq);
 551                 return ret;
 552         }
 553
 554         data->clk = devm_clk_get(dev, "sysmmu");
 555         if (IS_ERR(data->clk)) {
 556                 dev_err(dev, "Failed to get clock!\n");
 557                 return PTR_ERR(data->clk);
 558         } else  {
 559                 ret = clk_prepare(data->clk);
 560                 if (ret) {
 561                         dev_err(dev, "Failed to prepare clk\n");
 562                         return ret;
 563                 }
 564         }
 565
 566         data->clk_master = devm_clk_get(dev, "master");
 567         if (!IS_ERR(data->clk_master)) {
 568                 ret = clk_prepare(data->clk_master);
 569                 if (ret) {
 570                         clk_unprepare(data->clk);
 571                         dev_err(dev, "Failed to prepare master's clk\n");
 572                         return ret;
 573                 }
 574         }
 575
 576         data->sysmmu = dev;
 577         spin_lock_init(&data->lock);
 578
 579         platform_set_drvdata(pdev, data);
 580
 581         pm_runtime_enable(dev);
 582
 583         return 0;
 584 }
 585
 586 static const struct of_device_id sysmmu_of_match[] __initconst = {
 587         { .compatible   = "samsung,exynos-sysmmu", },
 588         { },
 589 };
 590
 591 static struct platform_driver exynos_sysmmu_driver __refdata = {
 592         .probe  = exynos_sysmmu_probe,
 593         .driver = {
 594                 .name           = "exynos-sysmmu",
 595                 .of_match_table = sysmmu_of_match,
 596         }
 597 };
 598
 599 static inline void pgtable_flush(void *vastart, void *vaend)
 600 {
 601         dmac_flush_range(vastart, vaend);
 602         outer_flush_range(virt_to_phys(vastart),
 603                                 virt_to_phys(vaend));
 604 }
 605
 606 static struct iommu_domain *exynos_iommu_domain_alloc(unsigned type)
 607 {
 608         struct exynos_iommu_domain *domain;
 609         int i;
 610
 611         if (type != IOMMU_DOMAIN_UNMANAGED)
 612                 return NULL;
 613
 614         domain = kzalloc(sizeof(*domain), GFP_KERNEL);
 615         if (!domain)
 616                 return NULL;
 617
 618         domain->pgtable = (sysmmu_pte_t *)__get_free_pages(GFP_KERNEL, 2);
 619         if (!domain->pgtable)
 620                 goto err_pgtable;
 621
 622         domain->lv2entcnt = (short *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1);
 623         if (!domain->lv2entcnt)
 624                 goto err_counter;
 625
 626         /* Workaround for System MMU v3.3 to prevent caching 1MiB mapping */
 627         for (i = 0; i < NUM_LV1ENTRIES; i += 8) {
 628                 domain->pgtable[i + 0] = ZERO_LV2LINK;
 629                 domain->pgtable[i + 1] = ZERO_LV2LINK;
 630                 domain->pgtable[i + 2] = ZERO_LV2LINK;
 631                 domain->pgtable[i + 3] = ZERO_LV2LINK;
 632                 domain->pgtable[i + 4] = ZERO_LV2LINK;
 633                 domain->pgtable[i + 5] = ZERO_LV2LINK;
 634                 domain->pgtable[i + 6] = ZERO_LV2LINK;
 635                 domain->pgtable[i + 7] = ZERO_LV2LINK;
 636         }
 637
 638         pgtable_flush(domain->pgtable, domain->pgtable + NUM_LV1ENTRIES);
 639
 640         spin_lock_init(&domain->lock);
 641         spin_lock_init(&domain->pgtablelock);
 642         INIT_LIST_HEAD(&domain->clients);
 643
 644         domain->domain.geometry.aperture_start = 0;
 645         domain->domain.geometry.aperture_end   = ~0UL;
 646         domain->domain.geometry.force_aperture = true;
 647
 648         return &domain->domain;
 649
 650 err_counter:
 651         free_pages((unsigned long)domain->pgtable, 2);
 652 err_pgtable:
 653         kfree(domain);
 654         return NULL;
 655 }
 656
 657 static void exynos_iommu_domain_free(struct iommu_domain *iommu_domain)
 658 {
 659         struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain);
 660         struct sysmmu_drvdata *data, *next;
 661         unsigned long flags;
 662         int i;
 663
 664         WARN_ON(!list_empty(&domain->clients));
 665
 666         spin_lock_irqsave(&domain->lock, flags);
 667
 668         list_for_each_entry_safe(data, next, &domain->clients, domain_node) {
 669                 if (__sysmmu_disable(data))
 670                         data->master = NULL;
 671                 list_del_init(&data->domain_node);
 672         }
 673
 674         spin_unlock_irqrestore(&domain->lock, flags);
 675
 676         for (i = 0; i < NUM_LV1ENTRIES; i++)
 677                 if (lv1ent_page(domain->pgtable + i))
 678                         kmem_cache_free(lv2table_kmem_cache,
 679                                 phys_to_virt(lv2table_base(domain->pgtable + i)));
 680
 681         free_pages((unsigned long)domain->pgtable, 2);
 682         free_pages((unsigned long)domain->lv2entcnt, 1);
 683         kfree(domain);
 684 }
 685
 686 static int exynos_iommu_attach_device(struct iommu_domain *iommu_domain,
 687                                    struct device *dev)
 688 {
 689         struct exynos_iommu_owner *owner = dev->archdata.iommu;
 690         struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain);
 691         struct sysmmu_drvdata *data;
 692         phys_addr_t pagetable = virt_to_phys(domain->pgtable);
 693         unsigned long flags;
 694         int ret = -ENODEV;
 695
 696         if (!has_sysmmu(dev))
 697                 return -ENODEV;
 698
 699         data = dev_get_drvdata(owner->sysmmu);
 700         if (data) {
 701                 ret = __sysmmu_enable(data, pagetable, domain);
 702                 if (ret >= 0) {
 703                         data->master = dev;
 704
 705                         spin_lock_irqsave(&domain->lock, flags);
 706                         list_add_tail(&data->domain_node, &domain->clients);
 707                         spin_unlock_irqrestore(&domain->lock, flags);
 708                 }
 709         }
 710
 711         if (ret < 0) {
 712                 dev_err(dev, "%s: Failed to attach IOMMU with pgtable %pa\n",
 713                                         __func__, &pagetable);
 714                 return ret;
 715         }
 716
 717         dev_dbg(dev, "%s: Attached IOMMU with pgtable %pa %s\n",
 718                 __func__, &pagetable, (ret == 0) ? "" : ", again");
 719
 720         return ret;
 721 }
 722
 723 static void exynos_iommu_detach_device(struct iommu_domain *iommu_domain,
 724                                     struct device *dev)
 725 {
 726         struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain);
 727         phys_addr_t pagetable = virt_to_phys(domain->pgtable);
 728         struct sysmmu_drvdata *data;
 729         unsigned long flags;
 730         bool found = false;
 731
 732         if (!has_sysmmu(dev))
 733                 return;
 734
 735         spin_lock_irqsave(&domain->lock, flags);
 736         list_for_each_entry(data, &domain->clients, domain_node) {
 737                 if (data->master == dev) {
 738                         if (__sysmmu_disable(data)) {
 739                                 data->master = NULL;
 740                                 list_del_init(&data->domain_node);
 741                         }
 742                         found = true;
 743                         break;
 744                 }
 745         }
 746         spin_unlock_irqrestore(&domain->lock, flags);
 747
 748         if (found)
 749                 dev_dbg(dev, "%s: Detached IOMMU with pgtable %pa\n",
 750                                         __func__, &pagetable);
 751         else
 752                 dev_err(dev, "%s: No IOMMU is attached\n", __func__);
 753 }
 754
 755 static sysmmu_pte_t *alloc_lv2entry(struct exynos_iommu_domain *domain,
 756                 sysmmu_pte_t *sent, sysmmu_iova_t iova, short *pgcounter)
 757 {
 758         if (lv1ent_section(sent)) {
 759                 WARN(1, "Trying mapping on %#08x mapped with 1MiB page", iova);
 760                 return ERR_PTR(-EADDRINUSE);
 761         }
 762
 763         if (lv1ent_fault(sent)) {
 764                 sysmmu_pte_t *pent;
 765                 bool need_flush_flpd_cache = lv1ent_zero(sent);
 766
 767                 pent = kmem_cache_zalloc(lv2table_kmem_cache, GFP_ATOMIC);
 768                 BUG_ON((unsigned int)pent & (LV2TABLE_SIZE - 1));
 769                 if (!pent)
 770                         return ERR_PTR(-ENOMEM);
 771
 772                 *sent = mk_lv1ent_page(virt_to_phys(pent));
 773                 kmemleak_ignore(pent);
 774                 *pgcounter = NUM_LV2ENTRIES;
 775                 pgtable_flush(pent, pent + NUM_LV2ENTRIES);
 776                 pgtable_flush(sent, sent + 1);
 777
 778                 /*
 779                  * If pre-fetched SLPD is a faulty SLPD in zero_l2_table,
 780                  * FLPD cache may cache the address of zero_l2_table. This
 781                  * function replaces the zero_l2_table with new L2 page table
 782                  * to write valid mappings.
 783                  * Accessing the valid area may cause page fault since FLPD
 784                  * cache may still cache zero_l2_table for the valid area
 785                  * instead of new L2 page table that has the mapping
 786                  * information of the valid area.
 787                  * Thus any replacement of zero_l2_table with other valid L2
 788                  * page table must involve FLPD cache invalidation for System
 789                  * MMU v3.3.
 790                  * FLPD cache invalidation is performed with TLB invalidation
 791                  * by VPN without blocking. It is safe to invalidate TLB without
 792                  * blocking because the target address of TLB invalidation is
 793                  * not currently mapped.
 794                  */
 795                 if (need_flush_flpd_cache) {
 796                         struct sysmmu_drvdata *data;
 797
 798                         spin_lock(&domain->lock);
 799                         list_for_each_entry(data, &domain->clients, domain_node)
 800                                 sysmmu_tlb_invalidate_flpdcache(data, iova);
 801                         spin_unlock(&domain->lock);
 802                 }
 803         }
 804
 805         return page_entry(sent, iova);
 806 }
 807
 808 static int lv1set_section(struct exynos_iommu_domain *domain,
 809                           sysmmu_pte_t *sent, sysmmu_iova_t iova,
 810                           phys_addr_t paddr, short *pgcnt)
 811 {
 812         if (lv1ent_section(sent)) {
 813                 WARN(1, "Trying mapping on 1MiB@%#08x that is mapped",
 814                         iova);
 815                 return -EADDRINUSE;
 816         }
 817
 818         if (lv1ent_page(sent)) {
 819                 if (*pgcnt != NUM_LV2ENTRIES) {
 820                         WARN(1, "Trying mapping on 1MiB@%#08x that is mapped",
 821                                 iova);
 822                         return -EADDRINUSE;
 823                 }
 824
 825                 kmem_cache_free(lv2table_kmem_cache, page_entry(sent, 0));
 826                 *pgcnt = 0;
 827         }
 828
 829         *sent = mk_lv1ent_sect(paddr);
 830
 831         pgtable_flush(sent, sent + 1);
 832
 833         spin_lock(&domain->lock);
 834         if (lv1ent_page_zero(sent)) {
 835                 struct sysmmu_drvdata *data;
 836                 /*
 837                  * Flushing FLPD cache in System MMU v3.3 that may cache a FLPD
 838                  * entry by speculative prefetch of SLPD which has no mapping.
 839                  */
 840                 list_for_each_entry(data, &domain->clients, domain_node)
 841                         sysmmu_tlb_invalidate_flpdcache(data, iova);
 842         }
 843         spin_unlock(&domain->lock);
 844
 845         return 0;
 846 }
 847
 848 static int lv2set_page(sysmmu_pte_t *pent, phys_addr_t paddr, size_t size,
 849                                                                 short *pgcnt)
 850 {
 851         if (size == SPAGE_SIZE) {
 852                 if (WARN_ON(!lv2ent_fault(pent)))
 853                         return -EADDRINUSE;
 854
 855                 *pent = mk_lv2ent_spage(paddr);
 856                 pgtable_flush(pent, pent + 1);
 857                 *pgcnt -= 1;
 858         } else { /* size == LPAGE_SIZE */
 859                 int i;
 860
 861                 for (i = 0; i < SPAGES_PER_LPAGE; i++, pent++) {
 862                         if (WARN_ON(!lv2ent_fault(pent))) {
 863                                 if (i > 0)
 864                                         memset(pent - i, 0, sizeof(*pent) * i);
 865                                 return -EADDRINUSE;
 866                         }
 867
 868                         *pent = mk_lv2ent_lpage(paddr);
 869                 }
 870                 pgtable_flush(pent - SPAGES_PER_LPAGE, pent);
 871                 *pgcnt -= SPAGES_PER_LPAGE;
 872         }
 873
 874         return 0;
 875 }
 876
 877 /*
 878  * *CAUTION* to the I/O virtual memory managers that support exynos-iommu:
 879  *
 880  * System MMU v3.x has advanced logic to improve address translation
 881  * performance with caching more page table entries by a page table walk.
 882  * However, the logic has a bug that while caching faulty page table entries,
 883  * System MMU reports page fault if the cached fault entry is hit even though
 884  * the fault entry is updated to a valid entry after the entry is cached.
 885  * To prevent caching faulty page table entries which may be updated to valid
 886  * entries later, the virtual memory manager should care about the workaround
 887  * for the problem. The following describes the workaround.
 888  *
 889  * Any two consecutive I/O virtual address regions must have a hole of 128KiB
 890  * at maximum to prevent misbehavior of System MMU 3.x (workaround for h/w bug).
 891  *
 892  * Precisely, any start address of I/O virtual region must be aligned with
 893  * the following sizes for System MMU v3.1 and v3.2.
 894  * System MMU v3.1: 128KiB
 895  * System MMU v3.2: 256KiB
 896  *
 897  * Because System MMU v3.3 caches page table entries more aggressively, it needs
 898  * more workarounds.
 899  * - Any two consecutive I/O virtual regions must have a hole of size larger
 900  *   than or equal to 128KiB.
 901  * - Start address of an I/O virtual region must be aligned by 128KiB.
 902  */
 903 static int exynos_iommu_map(struct iommu_domain *iommu_domain,
 904                             unsigned long l_iova, phys_addr_t paddr, size_t size,
 905                             int prot)
 906 {
 907         struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain);
 908         sysmmu_pte_t *entry;
 909         sysmmu_iova_t iova = (sysmmu_iova_t)l_iova;
 910         unsigned long flags;
 911         int ret = -ENOMEM;
 912
 913         BUG_ON(domain->pgtable == NULL);
 914
 915         spin_lock_irqsave(&domain->pgtablelock, flags);
 916
 917         entry = section_entry(domain->pgtable, iova);
 918
 919         if (size == SECT_SIZE) {
 920                 ret = lv1set_section(domain, entry, iova, paddr,
 921                                      &domain->lv2entcnt[lv1ent_offset(iova)]);
 922         } else {
 923                 sysmmu_pte_t *pent;
 924
 925                 pent = alloc_lv2entry(domain, entry, iova,
 926                                       &domain->lv2entcnt[lv1ent_offset(iova)]);
 927
 928                 if (IS_ERR(pent))
 929                         ret = PTR_ERR(pent);
 930                 else
 931                         ret = lv2set_page(pent, paddr, size,
 932                                        &domain->lv2entcnt[lv1ent_offset(iova)]);
 933         }
 934
 935         if (ret)
 936                 pr_err("%s: Failed(%d) to map %#zx bytes @ %#x\n",
 937                         __func__, ret, size, iova);
 938
 939         spin_unlock_irqrestore(&domain->pgtablelock, flags);
 940
 941         return ret;
 942 }
 943
 944 static void exynos_iommu_tlb_invalidate_entry(struct exynos_iommu_domain *domain,
 945                                               sysmmu_iova_t iova, size_t size)
 946 {
 947         struct sysmmu_drvdata *data;
 948         unsigned long flags;
 949
 950         spin_lock_irqsave(&domain->lock, flags);
 951
 952         list_for_each_entry(data, &domain->clients, domain_node)
 953                 sysmmu_tlb_invalidate_entry(data, iova, size);
 954
 955         spin_unlock_irqrestore(&domain->lock, flags);
 956 }
 957
 958 static size_t exynos_iommu_unmap(struct iommu_domain *iommu_domain,
 959                                  unsigned long l_iova, size_t size)
 960 {
 961         struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain);
 962         sysmmu_iova_t iova = (sysmmu_iova_t)l_iova;
 963         sysmmu_pte_t *ent;
 964         size_t err_pgsize;
 965         unsigned long flags;
 966
 967         BUG_ON(domain->pgtable == NULL);
 968
 969         spin_lock_irqsave(&domain->pgtablelock, flags);
 970
 971         ent = section_entry(domain->pgtable, iova);
 972
 973         if (lv1ent_section(ent)) {
 974                 if (WARN_ON(size < SECT_SIZE)) {
 975                         err_pgsize = SECT_SIZE;
 976                         goto err;
 977                 }
 978
 979                 /* workaround for h/w bug in System MMU v3.3 */
 980                 *ent = ZERO_LV2LINK;
 981                 pgtable_flush(ent, ent + 1);
 982                 size = SECT_SIZE;
 983                 goto done;
 984         }
 985
 986         if (unlikely(lv1ent_fault(ent))) {
 987                 if (size > SECT_SIZE)
 988                         size = SECT_SIZE;
 989                 goto done;
 990         }
 991
 992         /* lv1ent_page(sent) == true here */
 993
 994         ent = page_entry(ent, iova);
 995
 996         if (unlikely(lv2ent_fault(ent))) {
 997                 size = SPAGE_SIZE;
 998                 goto done;
 999         }
1000
1001         if (lv2ent_small(ent)) {
1002                 *ent = 0;
1003                 size = SPAGE_SIZE;
1004                 pgtable_flush(ent, ent + 1);
1005                 domain->lv2entcnt[lv1ent_offset(iova)] += 1;
1006                 goto done;
1007         }
1008
1009         /* lv1ent_large(ent) == true here */
1010         if (WARN_ON(size < LPAGE_SIZE)) {
1011                 err_pgsize = LPAGE_SIZE;
1012                 goto err;
1013         }
1014
1015         memset(ent, 0, sizeof(*ent) * SPAGES_PER_LPAGE);
1016         pgtable_flush(ent, ent + SPAGES_PER_LPAGE);
1017
1018         size = LPAGE_SIZE;
1019         domain->lv2entcnt[lv1ent_offset(iova)] += SPAGES_PER_LPAGE;
1020 done:
1021         spin_unlock_irqrestore(&domain->pgtablelock, flags);
1022
1023         exynos_iommu_tlb_invalidate_entry(domain, iova, size);
1024
1025         return size;
1026 err:
1027         spin_unlock_irqrestore(&domain->pgtablelock, flags);
1028
1029         pr_err("%s: Failed: size(%#zx) @ %#x is smaller than page size %#zx\n",
1030                 __func__, size, iova, err_pgsize);
1031
1032         return 0;
1033 }
1034
1035 static phys_addr_t exynos_iommu_iova_to_phys(struct iommu_domain *iommu_domain,
1036                                           dma_addr_t iova)
1037 {
1038         struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain);
1039         sysmmu_pte_t *entry;
1040         unsigned long flags;
1041         phys_addr_t phys = 0;
1042
1043         spin_lock_irqsave(&domain->pgtablelock, flags);
1044
1045         entry = section_entry(domain->pgtable, iova);
1046
1047         if (lv1ent_section(entry)) {
1048                 phys = section_phys(entry) + section_offs(iova);
1049         } else if (lv1ent_page(entry)) {
1050                 entry = page_entry(entry, iova);
1051
1052                 if (lv2ent_large(entry))
1053                         phys = lpage_phys(entry) + lpage_offs(iova);
1054                 else if (lv2ent_small(entry))
1055                         phys = spage_phys(entry) + spage_offs(iova);
1056         }
1057
1058         spin_unlock_irqrestore(&domain->pgtablelock, flags);
1059
1060         return phys;
1061 }
1062
1063 static int exynos_iommu_add_device(struct device *dev)
1064 {
1065         struct iommu_group *group;
1066         int ret;
1067
1068         group = iommu_group_get(dev);
1069
1070         if (!group) {
1071                 group = iommu_group_alloc();
1072                 if (IS_ERR(group)) {
1073                         dev_err(dev, "Failed to allocate IOMMU group\n");
1074                         return PTR_ERR(group);
1075                 }
1076         }
1077
1078         ret = iommu_group_add_device(group, dev);
1079         iommu_group_put(group);
1080
1081         return ret;
1082 }
1083
1084 static void exynos_iommu_remove_device(struct device *dev)
1085 {
1086         iommu_group_remove_device(dev);
1087 }
1088
1089 static const struct iommu_ops exynos_iommu_ops = {
1090         .domain_alloc = exynos_iommu_domain_alloc,
1091         .domain_free = exynos_iommu_domain_free,
1092         .attach_dev = exynos_iommu_attach_device,
1093         .detach_dev = exynos_iommu_detach_device,
1094         .map = exynos_iommu_map,
1095         .unmap = exynos_iommu_unmap,
1096         .map_sg = default_iommu_map_sg,
1097         .iova_to_phys = exynos_iommu_iova_to_phys,
1098         .add_device = exynos_iommu_add_device,
1099         .remove_device = exynos_iommu_remove_device,
1100         .pgsize_bitmap = SECT_SIZE | LPAGE_SIZE | SPAGE_SIZE,
1101 };
1102
1103 static int __init exynos_iommu_init(void)
1104 {
1105         struct device_node *np;
1106         int ret;
1107
1108         np = of_find_matching_node(NULL, sysmmu_of_match);
1109         if (!np)
1110                 return 0;
1111
1112         of_node_put(np);
1113
1114         lv2table_kmem_cache = kmem_cache_create("exynos-iommu-lv2table",
1115                                 LV2TABLE_SIZE, LV2TABLE_SIZE, 0, NULL);
1116         if (!lv2table_kmem_cache) {
1117                 pr_err("%s: Failed to create kmem cache\n", __func__);
1118                 return -ENOMEM;
1119         }
1120
1121         ret = platform_driver_register(&exynos_sysmmu_driver);
1122         if (ret) {
1123                 pr_err("%s: Failed to register driver\n", __func__);
1124                 goto err_reg_driver;
1125         }
1126
1127         zero_lv2_table = kmem_cache_zalloc(lv2table_kmem_cache, GFP_KERNEL);
1128         if (zero_lv2_table == NULL) {
1129                 pr_err("%s: Failed to allocate zero level2 page table\n",
1130                         __func__);
1131                 ret = -ENOMEM;
1132                 goto err_zero_lv2;
1133         }
1134
1135         ret = bus_set_iommu(&platform_bus_type, &exynos_iommu_ops);
1136         if (ret) {
1137                 pr_err("%s: Failed to register exynos-iommu driver.\n",
1138                                                                 __func__);
1139                 goto err_set_iommu;
1140         }
1141
1142         return 0;
1143 err_set_iommu:
1144         kmem_cache_free(lv2table_kmem_cache, zero_lv2_table);
1145 err_zero_lv2:
1146         platform_driver_unregister(&exynos_sysmmu_driver);
1147 err_reg_driver:
1148         kmem_cache_destroy(lv2table_kmem_cache);
1149         return ret;
1150 }
1151 subsys_initcall(exynos_iommu_init);